From 0665d6f07edeef5fadd0f84eadedf1b9de40242f Mon Sep 17 00:00:00 2001 From: Sudharsan Dhamal Gopalarathnam Date: Fri, 29 Oct 2021 15:12:27 -0700 Subject: [PATCH] VxLAN Tunnel Counters and Rates implementation (#1748) * Vxlan Tunnel counters implementation --- clear/main.py | 6 + counterpoll/main.py | 36 +++ scripts/tunnelstat | 342 +++++++++++++++++++++++++++++ setup.py | 1 + show/vxlan.py | 15 ++ tests/mock_tables/counters_db.json | 18 ++ tests/tunnelstat_test.py | 108 +++++++++ 7 files changed, 526 insertions(+) create mode 100755 scripts/tunnelstat create mode 100644 tests/tunnelstat_test.py diff --git a/clear/main.py b/clear/main.py index 8f93597b68da..5cbaee9d6265 100755 --- a/clear/main.py +++ b/clear/main.py @@ -193,6 +193,12 @@ def dropcounters(): command = "dropstat -c clear" run_command(command) +@cli.command() +def tunnelcounters(): + """Clear Tunnel counters""" + command = "tunnelstat -c" + run_command(command) + # # 'clear watermarks # diff --git a/counterpoll/main.py b/counterpoll/main.py index 7c062e9d747f..e04575d225ef 100644 --- a/counterpoll/main.py +++ b/counterpoll/main.py @@ -241,6 +241,39 @@ def disable(): configdb.mod_entry("FLEX_COUNTER_TABLE", "PG_WATERMARK", fc_info) configdb.mod_entry("FLEX_COUNTER_TABLE", BUFFER_POOL_WATERMARK, fc_info) +# Tunnel counter commands +@cli.group() +def tunnel(): + """ Tunnel counter commands """ + +@tunnel.command() +@click.argument('poll_interval', type=click.IntRange(100, 30000)) +def interval(poll_interval): + """ Set tunnel counter query interval """ + configdb = ConfigDBConnector() + configdb.connect() + tunnel_info = {} + tunnel_info['POLL_INTERVAL'] = poll_interval + configdb.mod_entry("FLEX_COUNTER_TABLE", "TUNNEL", tunnel_info) + +@tunnel.command() +def enable(): + """ Enable tunnel counter query """ + configdb = ConfigDBConnector() + configdb.connect() + tunnel_info = {} + tunnel_info['FLEX_COUNTER_STATUS'] = ENABLE + configdb.mod_entry("FLEX_COUNTER_TABLE", "TUNNEL", tunnel_info) + +@tunnel.command() +def disable(): + """ Disable tunnel counter query """ + configdb = ConfigDBConnector() + configdb.connect() + tunnel_info = {} + tunnel_info['FLEX_COUNTER_STATUS'] = DISABLE + configdb.mod_entry("FLEX_COUNTER_TABLE", "TUNNEL", tunnel_info) + @cli.command() def show(): """ Show the counter configuration """ @@ -254,6 +287,7 @@ def show(): pg_wm_info = configdb.get_entry('FLEX_COUNTER_TABLE', 'PG_WATERMARK') pg_drop_info = configdb.get_entry('FLEX_COUNTER_TABLE', PG_DROP) buffer_pool_wm_info = configdb.get_entry('FLEX_COUNTER_TABLE', BUFFER_POOL_WATERMARK) + tunnel_info = configdb.get_entry('FLEX_COUNTER_TABLE', 'TUNNEL') header = ("Type", "Interval (in ms)", "Status") data = [] @@ -273,6 +307,8 @@ def show(): data.append(['PG_DROP_STAT', pg_drop_info.get("POLL_INTERVAL", DEFLT_10_SEC), pg_drop_info.get("FLEX_COUNTER_STATUS", DISABLE)]) if buffer_pool_wm_info: data.append(["BUFFER_POOL_WATERMARK_STAT", buffer_pool_wm_info.get("POLL_INTERVAL", DEFLT_10_SEC), buffer_pool_wm_info.get("FLEX_COUNTER_STATUS", DISABLE)]) + if tunnel_info: + data.append(["TUNNEL_STAT", rif_info.get("POLL_INTERVAL", DEFLT_10_SEC), rif_info.get("FLEX_COUNTER_STATUS", DISABLE)]) click.echo(tabulate(data, headers=header, tablefmt="simple", missingval="")) diff --git a/scripts/tunnelstat b/scripts/tunnelstat new file mode 100755 index 000000000000..00aab5d8323d --- /dev/null +++ b/scripts/tunnelstat @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 + +##################################################################### +# +# tunnelstat is a tool for summarizing various tunnel statistics. +# +##################################################################### + +import _pickle as pickle +import argparse +import datetime +import sys +import os +import sys +import time + +# mock the redis for unit test purposes # +try: + if os.environ["UTILITIES_UNIT_TESTING"] == "2": + modules_path = os.path.join(os.path.dirname(__file__), "..") + test_path = os.path.join(modules_path, "tests") + sys.path.insert(0, modules_path) + sys.path.insert(0, test_path) + import mock_tables.dbconnector +except KeyError: + pass + +from collections import namedtuple, OrderedDict +from natsort import natsorted +from tabulate import tabulate +from utilities_common.netstat import ns_diff, table_as_json, STATUS_NA, format_prate +from swsscommon.swsscommon import SonicV2Connector + + +nstat_fields = ("rx_b_ok", "rx_p_ok", "tx_b_ok", "tx_p_ok") +NStats = namedtuple("NStats", nstat_fields) + +header = ['IFACE', 'RX_PKTS', 'RX_BYTES', 'RX_PPS','TX_PKTS', 'TX_BYTES', 'TX_PPS'] + +rates_key_list = [ 'RX_BPS', 'RX_PPS', 'TX_BPS', 'TX_PPS'] +ratestat_fields = ("rx_bps", "rx_pps", "tx_bps", "tx_pps") +RateStats = namedtuple("RateStats", ratestat_fields) + + +counter_names = ( + 'SAI_TUNNEL_STAT_IN_OCTETS', + 'SAI_TUNNEL_STAT_IN_PACKETS', + 'SAI_TUNNEL_STAT_OUT_OCTETS', + 'SAI_TUNNEL_STAT_OUT_PACKETS' +) + +counter_types = { + "vxlan": "SAI_TUNNEL_TYPE_VXLAN" +} + +COUNTER_TABLE_PREFIX = "COUNTERS:" +RATES_TABLE_PREFIX = "RATES:" +COUNTERS_TUNNEL_NAME_MAP = "COUNTERS_TUNNEL_NAME_MAP" +COUNTERS_TUNNEL_TYPE_MAP = "COUNTERS_TUNNEL_TYPE_MAP" + + +class Tunnelstat(object): + def __init__(self): + self.db = SonicV2Connector(use_unix_socket_path=False) + self.db.connect(self.db.COUNTERS_DB) + self.db.connect(self.db.APPL_DB) + + def get_cnstat(self, tunnel=None, tun_type=None): + """ + Get the counters info from database. + """ + def get_counters(table_id): + """ + Get the counters from specific table. + """ + fields = [STATUS_NA] * (len(nstat_fields)) + for pos, counter_name in enumerate(counter_names): + full_table_id = COUNTER_TABLE_PREFIX + table_id + counter_data = self.db.get(self.db.COUNTERS_DB, full_table_id, counter_name) + if counter_data: + fields[pos] = str(counter_data) + cntr = NStats._make(fields) + return cntr + + def get_rates(table_id): + """ + Get the rates from specific table. + """ + fields = ["0","0","0","0"] + for pos, name in enumerate(rates_key_list): + full_table_id = RATES_TABLE_PREFIX + table_id + counter_data = self.db.get(self.db.COUNTERS_DB, full_table_id, name) + if counter_data is None: + fields[pos] = STATUS_NA + elif fields[pos] != STATUS_NA: + fields[pos] = float(counter_data) + cntr = RateStats._make(fields) + return cntr + + # Build a dictionary of the stats + cnstat_dict = OrderedDict() + ratestat_dict = OrderedDict() + cnstat_dict['time'] = datetime.datetime.now() + + # Get the info from database + counter_tunnel_name_map = self.db.get_all(self.db.COUNTERS_DB, COUNTERS_TUNNEL_NAME_MAP) + counter_tunnel_type_map = self.db.get_all(self.db.COUNTERS_DB, COUNTERS_TUNNEL_TYPE_MAP) + + if counter_tunnel_name_map is None: + print("No %s in the DB!" % COUNTERS_TUNNEL_NAME_MAP) + sys.exit(1) + + if counter_tunnel_type_map is None: + print("No %s in the DB!" % COUNTERS_TUNNEL_TYPE_MAP) + sys.exit(1) + + if tun_type and tun_type not in counter_types: + print("Unknown tunnel type %s" % tun_type) + sys.exit(1) + + if tunnel and not tunnel in counter_tunnel_name_map: + print("Interface %s missing from %s! Make sure it exists" % (tunnel, COUNTERS_TUNNEL_NAME_MAP)) + sys.exit(2) + + if tunnel: + if tun_type and counter_types[tun_type] != counter_tunnel_type_map[counter_tunnel_name_map[tunnel]]: + print("Mismtch in tunnel type. Requested type %s actual type %s" % ( + counter_types[tun_type], counter_tunnel_type_map[counter_tunnel_name_map[tunnel]])) + sys.exit(2) + cnstat_dict[tunnel] = get_counters(counter_tunnel_name_map[tunnel]) + ratestat_dict[tunnel] = get_rates(counter_tunnel_name_map[tunnel]) + return cnstat_dict, ratestat_dict + + for tunnel in natsorted(counter_tunnel_name_map): + if not tun_type or counter_types[tun_type] == counter_tunnel_type_map[counter_tunnel_name_map[tunnel]]: + cnstat_dict[tunnel] = get_counters(counter_tunnel_name_map[tunnel]) + ratestat_dict[tunnel] = get_rates(counter_tunnel_name_map[tunnel]) + return cnstat_dict, ratestat_dict + + def cnstat_print(self, cnstat_dict, ratestat_dict, use_json): + """ + Print the cnstat. + """ + table = [] + + for key, data in cnstat_dict.items(): + if key == 'time': + continue + + rates = ratestat_dict.get(key, RateStats._make([STATUS_NA] * len(rates_key_list))) + table.append((key, data.rx_p_ok, data.rx_b_ok, format_prate(rates.rx_pps), + data.tx_p_ok, data.tx_b_ok, format_prate(rates.tx_pps))) + + if use_json: + print(table_as_json(table, header)) + + else: + print(tabulate(table, header, tablefmt='simple', stralign='right')) + + def cnstat_diff_print(self, cnstat_new_dict, cnstat_old_dict, ratestat_dict, use_json): + """ + Print the difference between two cnstat results. + """ + + table = [] + + for key, cntr in cnstat_new_dict.items(): + if key == 'time': + continue + old_cntr = None + if key in cnstat_old_dict: + old_cntr = cnstat_old_dict.get(key) + + rates = ratestat_dict.get(key, RateStats._make([STATUS_NA] * len(rates_key_list))) + if old_cntr is not None: + table.append((key, + ns_diff(cntr.rx_p_ok, old_cntr.rx_p_ok), + ns_diff(cntr.rx_b_ok, old_cntr.rx_b_ok), + format_prate(rates.rx_pps), + ns_diff(cntr.tx_p_ok, old_cntr.tx_p_ok), + ns_diff(cntr.tx_b_ok, old_cntr.tx_b_ok), + format_prate(rates.tx_pps))) + else: + table.append((key, + cntr.rx_p_ok, + cntr.rx_b_ok, + format_prate(rates.rx_pps), + cntr.tx_p_ok, + cntr.tx_b_ok, + format_prate(rates.tx_pps))) + if use_json: + print(table_as_json(table, header)) + else: + print(tabulate(table, header, tablefmt='simple', stralign='right')) + + def cnstat_single_tunnel(self, tunnel, cnstat_new_dict, cnstat_old_dict): + + header = tunnel + '\n' + '-'*len(tunnel) + body = """ + RX: + %10s packets + %10s bytes + TX: + %10s packets + %10s bytes""" + + cntr = cnstat_new_dict.get(tunnel) + + if cnstat_old_dict: + old_cntr = cnstat_old_dict.get(tunnel) + if old_cntr: + body = body % (ns_diff(cntr.rx_p_ok, old_cntr.rx_p_ok), + ns_diff(cntr.rx_b_ok, old_cntr.rx_b_ok), + ns_diff(cntr.tx_p_ok, old_cntr.tx_p_ok), + ns_diff(cntr.tx_b_ok, old_cntr.tx_b_ok)) + else: + body = body % (cntr.rx_p_ok, cntr.rx_b_ok, cntr.tx_p_ok, cntr.tx_b_ok) + + print(header) + print(body) + + +def main(): + parser = argparse.ArgumentParser(description='Display the tunnels state and counters', + formatter_class=argparse.RawTextHelpFormatter, + epilog=""" + Port state: (U)-Up (D)-Down (X)-Disabled + Examples: + tunnelstat -c -t test + tunnelstat -t test + tunnelstat -d -t test + tunnelstat + tunnelstat -p 20 + tunnelstat -i Vlan1000 + """) + + parser.add_argument('-c', '--clear', action='store_true', help='Copy & clear stats') + parser.add_argument('-d', '--delete', action='store_true', help='Delete saved stats, either the uid or the specified tag') + parser.add_argument('-D', '--delete-all', action='store_true', help='Delete all saved stats') + parser.add_argument('-j', '--json', action='store_true', help='Display in JSON format') + parser.add_argument('-t', '--tag', type=str, help='Save stats with name TAG', default=None) + parser.add_argument('-i', '--tunnel', type=str, help='Show stats for a single tunnel', required=False) + parser.add_argument('-p', '--period', type=int, help='Display stats over a specified period (in seconds).', default=0) + parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.0') + parser.add_argument('-T', '--type', type=str, help ='Display Vxlan tunnel stats', required=False) + args = parser.parse_args() + + save_fresh_stats = args.clear + delete_saved_stats = args.delete + delete_all_stats = args.delete_all + use_json = args.json + tag_name = args.tag if args.tag else "" + uid = str(os.getuid()) + wait_time_in_seconds = args.period + tunnel_name = args.tunnel if args.tunnel else "" + tunnel_type = args.type if args.type else "" + + # fancy filename with dashes: uid-tag-tunnel / uid-tunnel / uid-tag etc + filename_components = [uid, tag_name] + cnstat_file = "-".join(filter(None, filename_components)) + + cnstat_dir = "/tmp/tunnelstat-" + uid + cnstat_fqn_file = cnstat_dir + "/" + cnstat_file + + if delete_all_stats: + # There is nothing to delete + if not os.path.isdir(cnstat_dir): + sys.exit(0) + + for file in os.listdir(cnstat_dir): + os.remove(cnstat_dir + "/" + file) + + try: + os.rmdir(cnstat_dir) + sys.exit(0) + except IOError as e: + print(e.errno, e) + sys.exit(e) + + if delete_saved_stats: + try: + os.remove(cnstat_fqn_file) + except IOError as e: + if e.errno != ENOENT: + print(e.errno, e) + sys.exit(1) + finally: + if os.listdir(cnstat_dir) == []: + os.rmdir(cnstat_dir) + sys.exit(0) + + tunnelstat = Tunnelstat() + cnstat_dict,ratestat_dict = tunnelstat.get_cnstat(tunnel=tunnel_name, tun_type=tunnel_type) + + # At this point, either we'll create a file or open an existing one. + if not os.path.exists(cnstat_dir): + try: + os.makedirs(cnstat_dir) + except IOError as e: + print(e.errno, e) + sys.exit(1) + + if save_fresh_stats: + try: + pickle.dump(cnstat_dict, open(cnstat_fqn_file, 'wb')) + except IOError as e: + sys.exit(e.errno) + else: + print("Cleared counters") + sys.exit(0) + + if wait_time_in_seconds == 0: + if os.path.isfile(cnstat_fqn_file): + try: + cnstat_cached_dict = pickle.load(open(cnstat_fqn_file, 'rb')) + print("Last cached time was " + str(cnstat_cached_dict.get('time'))) + if tunnel_name: + tunnelstat.cnstat_single_tunnel(tunnel_name, cnstat_dict, cnstat_cached_dict) + else: + tunnelstat.cnstat_diff_print(cnstat_dict, cnstat_cached_dict, ratestat_dict, use_json) + except IOError as e: + print(e.errno, e) + else: + if tag_name: + print("\nFile belonging to tag %s does not exist" % tag_name) + print("Did you run 'tunnelstat -c -t %s' to record the counters via tag %s?\n" % (tag_name, tag_name)) + else: + if tunnel_name: + tunnelstat.cnstat_single_tunnel(tunnel_name, cnstat_dict, None) + else: + tunnelstat.cnstat_print(cnstat_dict, ratestat_dict, use_json) + else: + #wait for the specified time and then gather the new stats and output the difference. + time.sleep(wait_time_in_seconds) + cnstat_new_dict, ratestat_dict = tunnelstat.get_cnstat(tunnel=tunnel_name, tun_type=tunnel_type) + if tunnel_name: + tunnelstat.cnstat_single_tunnel(tunnel_name, cnstat_new_dict, cnstat_dict) + else: + tunnelstat.cnstat_diff_print(cnstat_new_dict, cnstat_dict, ratestat_dict, use_json) + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index c28af558ab46..30e8c7bfd479 100644 --- a/setup.py +++ b/setup.py @@ -128,6 +128,7 @@ 'scripts/storyteller', 'scripts/syseeprom-to-json', 'scripts/tempershow', + 'scripts/tunnelstat', 'scripts/update_json.py', 'scripts/voqutil', 'scripts/warm-reboot', diff --git a/show/vxlan.py b/show/vxlan.py index bb20580689d3..398de798f95c 100644 --- a/show/vxlan.py +++ b/show/vxlan.py @@ -305,3 +305,18 @@ def remotemac(remote_vtep_ip, count): output += ('%s \n' % (str(num))) click.echo(output) +@vxlan.command() +@click.argument('tunnel', required=False) +@click.option('-p', '--period') +@click.option('--verbose', is_flag=True, help="Enable verbose output") +def counters(tunnel, period, verbose): + """Show VxLAN counters""" + + cmd = "tunnelstat -T vxlan" + if period is not None: + cmd += " -p {}".format(period) + if tunnel is not None: + cmd += " -i {}".format(tunnel) + + clicommon.run_command(cmd, display_cmd=verbose) + diff --git a/tests/mock_tables/counters_db.json b/tests/mock_tables/counters_db.json index 9ad472c03d39..053165079e61 100644 --- a/tests/mock_tables/counters_db.json +++ b/tests/mock_tables/counters_db.json @@ -520,6 +520,12 @@ "TX_BPS": "0", "TX_PPS": "0" }, + "COUNTERS:oid:0x2a0000000035e": { + "SAI_TUNNEL_STAT_IN_OCTETS": "81922", + "SAI_TUNNEL_STAT_IN_PACKETS": "452", + "SAI_TUNNEL_STAT_OUT_OCTETS": "23434", + "SAI_TUNNEL_STAT_OUT_PACKETS": "154" + }, "COUNTERS_RIF_NAME_MAP": { "Ethernet20": "oid:0x600000000065f", "PortChannel0001": "oid:0x60000000005a1", @@ -536,6 +542,18 @@ "oid:0x600000000063d": "SAI_ROUTER_INTERFACE_TYPE_PORT", "oid:0x600000000065f": "SAI_ROUTER_INTERFACE_TYPE_PORT" }, + "COUNTERS_TUNNEL_NAME_MAP": { + "vtep1": "oid:0x2a0000000035e" + }, + "COUNTERS_TUNNEL_TYPE_MAP": { + "oid:0x2a0000000035e": "SAI_TUNNEL_TYPE_VXLAN" + }, + "RATES:oid:0x2a0000000035e": { + "RX_BPS": "20971520", + "RX_PPS": "20523", + "TX_BPS": "2048", + "TX_PPS": "201" + }, "COUNTERS:DATAACL:DEFAULT_RULE": { "Bytes": "1", "Packets": "2" diff --git a/tests/tunnelstat_test.py b/tests/tunnelstat_test.py new file mode 100644 index 000000000000..f1fe716ef38b --- /dev/null +++ b/tests/tunnelstat_test.py @@ -0,0 +1,108 @@ +import sys +import os +import traceback + +from click.testing import CliRunner + +test_path = os.path.dirname(os.path.abspath(__file__)) +modules_path = os.path.dirname(test_path) +scripts_path = os.path.join(modules_path, "scripts") +sys.path.insert(0, test_path) +sys.path.insert(0, modules_path) + +import clear.main as clear +import show.main as show +from .mock_tables import dbconnector + +show_vxlan_counters_output = """\ + IFACE RX_PKTS RX_BYTES RX_PPS TX_PKTS TX_BYTES TX_PPS +------- --------- ---------- ---------- --------- ---------- -------- + vtep1 452 81922 20523.00/s 154 23434 201.00/s +""" + +show_vxlan_counters_clear_output = """\ + IFACE RX_PKTS RX_BYTES RX_PPS TX_PKTS TX_BYTES TX_PPS +------- --------- ---------- ---------- --------- ---------- -------- + vtep1 0 0 20523.00/s 0 0 201.00/s +""" + +show_vxlan_counters_interface_output = """\ +vtep1 +----- + + RX: + 452 packets + 81922 bytes + TX: + 154 packets + 23434 bytes +""" + +show_vxlan_counters_clear_interface_output = """\ +vtep1 +----- + + RX: + 0 packets + 0 bytes + TX: + 0 packets + 0 bytes +""" + + +class TestTunnelstat(object): + @classmethod + def setup_class(cls): + print("SETUP") + os.environ["PATH"] += os.pathsep + scripts_path + os.environ["UTILITIES_UNIT_TESTING"] = "2" + + def test_no_param(self): + runner = CliRunner() + result = runner.invoke(show.cli.commands["vxlan"].commands["counters"], []) + print(result.exit_code) + print(result.output) + traceback.print_tb(result.exc_info[2]) + assert result.exit_code == 0 + assert result.output == show_vxlan_counters_output + + def test_single_tunnel(self): + runner = CliRunner() + result = runner.invoke(show.cli.commands["vxlan"].commands["counters"], ["vtep1"]) + expected = show_vxlan_counters_interface_output + assert result.output == expected + + def test_clear(self): + runner = CliRunner() + result = runner.invoke(clear.cli.commands["tunnelcounters"], []) + print(result.stdout) + assert result.exit_code == 0 + result = runner.invoke(show.cli.commands["vxlan"].commands["counters"], []) + print(result.stdout) + expected = show_vxlan_counters_clear_output + + # remove the counters snapshot + show.run_command("tunnelstat -D") + for line in expected: + assert line in result.output + + def test_clear_interface(self): + runner = CliRunner() + result = runner.invoke(clear.cli.commands["tunnelcounters"], []) + print(result.stdout) + assert result.exit_code == 0 + result = runner.invoke(show.cli.commands["vxlan"].commands["counters"], ["vtep1"]) + print(result.stdout) + expected = show_vxlan_counters_clear_interface_output + + # remove the counters snapshot + show.run_command("tunnelstat -D") + for line in expected: + assert line in result.output + + @classmethod + def teardown_class(cls): + print("TEARDOWN") + os.environ["PATH"] = os.pathsep.join(os.environ["PATH"].split(os.pathsep)[:-1]) + os.environ["UTILITIES_UNIT_TESTING"] = "0"