Replace pickle by json (sonic-net#2849)

This reverts commit 10f31ea. Revert "Revert "Replace pickle by json (sonic-net#2636)" (sonic-net#2746)" #### What I did Revert PR 2746 Add fix `default=json_serial` in intfstat #### How I did it #### How to verify it ``` 1. dropstat a. dropstat -c clear b. dropstat -c show 2. flow_counter_stat a. sonic-clear flowcnt-trap b. show flowcnt-trap stats 3. intfstat a. intfstat -c b. intfstat -c c. intfstat -c -t test d. intfstat -c -t test e. intfstat -p 0 4. pfcstat a. pfcstat -c b. pfcstat -s frontend 5. pg-drop a. pg-drop -c clear b. pg-drop -c show 6. portstat a. portstat -s frontend b. portstat -c 7. queuestat a. queuestat -c b. queuestat c. queuestat -p Ethernet0 8. tunnelstat a. tunnelstat -c b. tunnelstat ```
pdhruv-marvell · Aug 23, 2023 · 20f316b · 20f316b
1 parent cc9ef65
commit 20f316b
Show file tree

Hide file tree

Showing 9 changed files with 258 additions and 237 deletions.
diff --git a/scripts/dropstat b/scripts/dropstat
@@ -11,7 +11,7 @@
 # - Refactor calls to COUNTERS_DB to reduce redundancy
 # - Cache DB queries to reduce # of expensive queries
 
-import _pickle as pickle
+import json
 import argparse
 import os
 import socket
@@ -117,10 +117,10 @@ class DropStat(object):
  """
 
  try:
- pickle.dump(self.get_counts_table(self.gather_counters(std_port_rx_counters + std_port_tx_counters, DEBUG_COUNTER_PORT_STAT_MAP), COUNTERS_PORT_NAME_MAP),
- open(self.port_drop_stats_file, 'wb+'))
- pickle.dump(self.get_counts(self.gather_counters([], DEBUG_COUNTER_SWITCH_STAT_MAP), self.get_switch_id()),
- open(self.switch_drop_stats_file, 'wb+'))
+ json.dump(self.get_counts_table(self.gather_counters(std_port_rx_counters + std_port_tx_counters, DEBUG_COUNTER_PORT_STAT_MAP), COUNTERS_PORT_NAME_MAP),
+ open(self.port_drop_stats_file, 'w+'))
+ json.dump(self.get_counts(self.gather_counters([], DEBUG_COUNTER_SWITCH_STAT_MAP), self.get_switch_id()),
+ open(self.switch_drop_stats_file, 'w+'))
  except IOError as e:
  print(e)
  sys.exit(e.errno)
@@ -135,7 +135,7 @@ class DropStat(object):
 
  # Grab the latest clear checkpoint, if it exists
  if os.path.isfile(self.port_drop_stats_file):
- port_drop_ckpt = pickle.load(open(self.port_drop_stats_file, 'rb'))
+ port_drop_ckpt = json.load(open(self.port_drop_stats_file, 'r'))
 
  counters = self.gather_counters(std_port_rx_counters + std_port_tx_counters, DEBUG_COUNTER_PORT_STAT_MAP, group, counter_type)
  headers = std_port_description_header + self.gather_headers(counters, DEBUG_COUNTER_PORT_STAT_MAP)
@@ -162,7 +162,7 @@ class DropStat(object):
 
  # Grab the latest clear checkpoint, if it exists
  if os.path.isfile(self.switch_drop_stats_file):
- switch_drop_ckpt = pickle.load(open(self.switch_drop_stats_file, 'rb'))
+ switch_drop_ckpt = json.load(open(self.switch_drop_stats_file, 'r'))
 
  counters = self.gather_counters([], DEBUG_COUNTER_SWITCH_STAT_MAP, group, counter_type)
  headers = std_switch_description_header + self.gather_headers(counters, DEBUG_COUNTER_SWITCH_STAT_MAP)

diff --git a/scripts/flow_counters_stat b/scripts/flow_counters_stat
@@ -2,7 +2,7 @@
 
 import argparse
 import os
-import _pickle as pickle
+import json
 import sys
 
 from natsort import natsorted
@@ -185,8 +185,8 @@ class FlowCounterStats(object):
  if os.path.exists(self.data_file):
  os.remove(self.data_file)
 
- with open(self.data_file, 'wb') as f:
- pickle.dump(data, f)
+ with open(self.data_file, 'w') as f:
+ json.dump(data, f)
  except IOError as e:
  print('Failed to save statistic - {}'.format(repr(e)))
 
@@ -200,8 +200,8 @@ class FlowCounterStats(object):
  return None
 
  try:
- with open(self.data_file, 'rb') as f:
- data = pickle.load(f)
+ with open(self.data_file, 'r') as f:
+ data = json.load(f)
  except IOError as e:
  print('Failed to load statistic - {}'.format(repr(e)))
  return None

diff --git a/scripts/intfstat b/scripts/intfstat
@@ -6,7 +6,7 @@
 #
 #####################################################################
 
-import _pickle as pickle
+import json
 import argparse
 import datetime
 import sys
@@ -28,7 +28,7 @@ from collections import namedtuple, OrderedDict
 from natsort import natsorted
 from tabulate import tabulate
 from utilities_common.netstat import ns_diff, table_as_json, STATUS_NA, format_brate, format_prate
-from utilities_common.cli import UserCache
+from utilities_common.cli import json_serial, UserCache
 from swsscommon.swsscommon import SonicV2Connector
 
 nstat_fields = (
@@ -96,7 +96,7 @@ class Intfstat(object):
  counter_data = self.db.get(self.db.COUNTERS_DB, full_table_id, counter_name)
  if counter_data:
  fields[pos] = str(counter_data)
- cntr = NStats._make(fields)
+ cntr = NStats._make(fields)._asdict()
  return cntr
 
  def get_rates(table_id):
@@ -153,14 +153,14 @@ class Intfstat(object):
  rates = ratestat_dict.get(key, RateStats._make([STATUS_NA] * len(rates_key_list)))
 
  table.append((key,
- data.rx_p_ok,
+ data['rx_p_ok'],
  format_brate(rates.rx_bps),
  format_prate(rates.rx_pps),
- data.rx_p_err,
- data.tx_p_ok,
+ data['rx_p_err'],
+ data['tx_p_ok'],
  format_brate(rates.tx_bps),
  format_prate(rates.tx_pps),
- data.tx_p_err))
+ data['tx_p_err']))
 
  if use_json:
  print(table_as_json(table, header))
@@ -186,24 +186,24 @@ class Intfstat(object):
 
  if old_cntr is not None:
  table.append((key,
- ns_diff(cntr.rx_p_ok, old_cntr.rx_p_ok),
+ ns_diff(cntr['rx_p_ok'], old_cntr['rx_p_ok']),
  format_brate(rates.rx_bps),
  format_prate(rates.rx_pps),
- ns_diff(cntr.rx_p_err, old_cntr.rx_p_err),
- ns_diff(cntr.tx_p_ok, old_cntr.tx_p_ok),
+ ns_diff(cntr['rx_p_err'], old_cntr['rx_p_err']),
+ ns_diff(cntr['tx_p_ok'], old_cntr['tx_p_ok']),
  format_brate(rates.tx_bps),
  format_prate(rates.tx_pps),
- ns_diff(cntr.tx_p_err, old_cntr.tx_p_err)))
+ ns_diff(cntr['tx_p_err'], old_cntr['tx_p_err'])))
  else:
  table.append((key,
- cntr.rx_p_ok,
+ cntr['rx_p_ok'],
  format_brate(rates.rx_bps),
  format_prate(rates.rx_pps),
- cntr.rx_p_err,
- cntr.tx_p_ok,
+ cntr['rx_p_err'],
+ cntr['tx_p_ok'],
  format_brate(rates.tx_bps),
  format_prate(rates.tx_pps),
- cntr.tx_p_err))
+ cntr['tx_p_err']))
 
  if use_json:
  print(table_as_json(table, header))
@@ -229,17 +229,17 @@ class Intfstat(object):
 
  if cnstat_old_dict and cnstat_old_dict.get(rif):
  old_cntr = cnstat_old_dict.get(rif)
- body = body % (ns_diff(cntr.rx_p_ok, old_cntr.rx_p_ok),
- ns_diff(cntr.rx_b_ok, old_cntr.rx_b_ok),
- ns_diff(cntr.rx_p_err, old_cntr.rx_p_err),
- ns_diff(cntr.rx_b_err, old_cntr.rx_b_err),
- ns_diff(cntr.tx_p_ok, old_cntr.tx_p_ok),
- ns_diff(cntr.tx_b_ok, old_cntr.tx_b_ok),
- ns_diff(cntr.tx_p_err, old_cntr.tx_p_err),
- ns_diff(cntr.tx_b_err, old_cntr.tx_b_err))
+ body = body % (ns_diff(cntr['rx_p_ok'], old_cntr['rx_p_ok']),
+ ns_diff(cntr['rx_b_ok'], old_cntr['rx_b_ok']),
+ ns_diff(cntr['rx_p_err'], old_cntr['rx_p_err']),
+ ns_diff(cntr['rx_b_err'], old_cntr['rx_b_err']),
+ ns_diff(cntr['tx_p_ok'], old_cntr['tx_p_ok']),
+ ns_diff(cntr['tx_b_ok'], old_cntr['tx_b_ok']),
+ ns_diff(cntr['tx_p_err'], old_cntr['tx_p_err']),
+ ns_diff(cntr['tx_b_err'], old_cntr['tx_b_err']))
  else:
- body = body % (cntr.rx_p_ok, cntr.rx_b_ok, cntr.rx_p_err,cntr.rx_b_err,
- cntr.tx_p_ok, cntr.tx_b_ok, cntr.tx_p_err, cntr.tx_b_err)
+ body = body % (cntr['rx_p_ok'], cntr['rx_b_ok'], cntr['rx_p_err'],cntr['rx_b_err'],
+ cntr['tx_p_ok'], cntr['tx_b_ok'], cntr['tx_p_err'], cntr['tx_b_err'])
 
  print(header)
  print(body)
@@ -305,20 +305,20 @@ def main():
  if tag_name is not None:
  if os.path.isfile(cnstat_fqn_general_file):
  try:
- general_data = pickle.load(open(cnstat_fqn_general_file, 'rb'))
+ general_data = json.load(open(cnstat_fqn_general_file, 'r'))
  for key, val in cnstat_dict.items():
  general_data[key] = val
- pickle.dump(general_data, open(cnstat_fqn_general_file, 'wb'))
+ json.dump(general_data, open(cnstat_fqn_general_file, 'w'), default=json_serial)
  except IOError as e:
  sys.exit(e.errno)
  # Add the information also to tag specific file
  if os.path.isfile(cnstat_fqn_file):
- data = pickle.load(open(cnstat_fqn_file, 'rb'))
+ data = json.load(open(cnstat_fqn_file, 'r'))
  for key, val in cnstat_dict.items():
  data[key] = val
- pickle.dump(data, open(cnstat_fqn_file, 'wb'))
+ json.dump(data, open(cnstat_fqn_file, 'w'), default=json_serial)
  else:
- pickle.dump(cnstat_dict, open(cnstat_fqn_file, 'wb'))
+ json.dump(cnstat_dict, open(cnstat_fqn_file, 'w'), default=json_serial)
  except IOError as e:
  sys.exit(e.errno)
  else:
@@ -330,9 +330,9 @@ def main():
  try:
  cnstat_cached_dict = {}
  if os.path.isfile(cnstat_fqn_file):
- cnstat_cached_dict = pickle.load(open(cnstat_fqn_file, 'rb'))
+ cnstat_cached_dict = json.load(open(cnstat_fqn_file, 'r'))
  else:
- cnstat_cached_dict = pickle.load(open(cnstat_fqn_general_file, 'rb'))
+ cnstat_cached_dict = json.load(open(cnstat_fqn_general_file, 'r'))
 
  print("Last cached time was " + str(cnstat_cached_dict.get('time')))
  if interface_name:

diff --git a/scripts/pfcstat b/scripts/pfcstat
@@ -6,7 +6,7 @@
 #
 #####################################################################
 
-import _pickle as pickle
+import json
 import argparse
 import datetime
 import os.path
@@ -37,7 +37,7 @@ except KeyError:
 from utilities_common.netstat import ns_diff, STATUS_NA, format_number_with_comma
 from utilities_common import multi_asic as multi_asic_util
 from utilities_common import constants
-from utilities_common.cli import UserCache
+from utilities_common.cli import json_serial, UserCache
 
 
 PStats = namedtuple("PStats", "pfc0, pfc1, pfc2, pfc3, pfc4, pfc5, pfc6, pfc7")
@@ -101,7 +101,7 @@ class Pfcstat(object):
  fields[pos] = STATUS_NA
  else:
  fields[pos] = str(int(counter_data))
- cntr = PStats._make(fields)
+ cntr = PStats._make(fields)._asdict()
  return cntr
 
  # Get the info from database
@@ -144,14 +144,14 @@ class Pfcstat(object):
  if key == 'time':
  continue
  table.append((key,
- format_number_with_comma(data.pfc0),
- format_number_with_comma(data.pfc1),
- format_number_with_comma(data.pfc2),
- format_number_with_comma(data.pfc3),
- format_number_with_comma(data.pfc4),
- format_number_with_comma(data.pfc5),
- format_number_with_comma(data.pfc6),
- format_number_with_comma(data.pfc7)))
+ format_number_with_comma(data['pfc0']),
+ format_number_with_comma(data['pfc1']),
+ format_number_with_comma(data['pfc2']),
+ format_number_with_comma(data['pfc3']),
+ format_number_with_comma(data['pfc4']),
+ format_number_with_comma(data['pfc5']),
+ format_number_with_comma(data['pfc6']),
+ format_number_with_comma(data['pfc7'])))
 
  if rx:
  print(tabulate(table, header_Rx, tablefmt='simple', stralign='right'))
@@ -173,24 +173,24 @@ class Pfcstat(object):
 
  if old_cntr is not None:
  table.append((key,
- ns_diff(cntr.pfc0, old_cntr.pfc0),
- ns_diff(cntr.pfc1, old_cntr.pfc1),
- ns_diff(cntr.pfc2, old_cntr.pfc2),
- ns_diff(cntr.pfc3, old_cntr.pfc3),
- ns_diff(cntr.pfc4, old_cntr.pfc4),
- ns_diff(cntr.pfc5, old_cntr.pfc5),
- ns_diff(cntr.pfc6, old_cntr.pfc6),
- ns_diff(cntr.pfc7, old_cntr.pfc7)))
+ ns_diff(cntr['pfc0'], old_cntr['pfc0']),
+ ns_diff(cntr['pfc1'], old_cntr['pfc1']),
+ ns_diff(cntr['pfc2'], old_cntr['pfc2']),
+ ns_diff(cntr['pfc3'], old_cntr['pfc3']),
+ ns_diff(cntr['pfc4'], old_cntr['pfc4']),
+ ns_diff(cntr['pfc5'], old_cntr['pfc5']),
+ ns_diff(cntr['pfc6'], old_cntr['pfc6']),
+ ns_diff(cntr['pfc7'], old_cntr['pfc7'])))
  else:
  table.append((key,
- format_number_with_comma(cntr.pfc0),
- format_number_with_comma(cntr.pfc1),
- format_number_with_comma(cntr.pfc2),
- format_number_with_comma(cntr.pfc3),
- format_number_with_comma(cntr.pfc4),
- format_number_with_comma(cntr.pfc5),
- format_number_with_comma(cntr.pfc6),
- format_number_with_comma(cntr.pfc7)))
+ format_number_with_comma(cntr['pfc0']),
+ format_number_with_comma(cntr['pfc1']),
+ format_number_with_comma(cntr['pfc2']),
+ format_number_with_comma(cntr['pfc3']),
+ format_number_with_comma(cntr['pfc4']),
+ format_number_with_comma(cntr['pfc5']),
+ format_number_with_comma(cntr['pfc6']),
+ format_number_with_comma(cntr['pfc7'])))
 
  if rx:
  print(tabulate(table, header_Rx, tablefmt='simple', stralign='right'))
@@ -256,8 +256,8 @@ Examples:
 
  if save_fresh_stats:
  try:
- pickle.dump(cnstat_dict_rx, open(cnstat_fqn_file_rx, 'wb'))
- pickle.dump(cnstat_dict_tx, open(cnstat_fqn_file_tx, 'wb'))
+ json.dump(cnstat_dict_rx, open(cnstat_fqn_file_rx, 'w'), default=json_serial)
+ json.dump(cnstat_dict_tx, open(cnstat_fqn_file_tx, 'w'), default=json_serial)
  except IOError as e:
  print(e.errno, e)
  sys.exit(e.errno)
@@ -271,7 +271,7 @@ Examples:
  """
  if os.path.isfile(cnstat_fqn_file_rx):
  try:
- cnstat_cached_dict = pickle.load(open(cnstat_fqn_file_rx, 'rb'))
+ cnstat_cached_dict = json.load(open(cnstat_fqn_file_rx, 'r'))
  print("Last cached time was " + str(cnstat_cached_dict.get('time')))
  pfcstat.cnstat_diff_print(cnstat_dict_rx, cnstat_cached_dict, True)
  except IOError as e:
@@ -286,7 +286,7 @@ Examples:
  """
  if os.path.isfile(cnstat_fqn_file_tx):
  try:
- cnstat_cached_dict = pickle.load(open(cnstat_fqn_file_tx, 'rb'))
+ cnstat_cached_dict = json.load(open(cnstat_fqn_file_tx, 'r'))
  print("Last cached time was " + str(cnstat_cached_dict.get('time')))
  pfcstat.cnstat_diff_print(cnstat_dict_tx, cnstat_cached_dict, False)
  except IOError as e:

diff --git a/scripts/pg-drop b/scripts/pg-drop
@@ -5,7 +5,7 @@
 # pg-drop is a tool for show/clear ingress pg dropped packet stats.
 #
 #####################################################################
-import _pickle as pickle
+import json
 import argparse
 import os
 import sys
@@ -144,7 +144,7 @@ class PgDropStat(object):
  port_drop_ckpt = {}
  # Grab the latest clear checkpoint, if it exists
  if os.path.isfile(self.port_drop_stats_file):
- port_drop_ckpt = pickle.load(open(self.port_drop_stats_file, 'rb'))
+ port_drop_ckpt = json.load(open(self.port_drop_stats_file, 'r'))
 
  # Header list contains the port name followed by the PGs. Fields is used to populate the pg values
  fields = ["0"]* (len(self.header_list) - 1)
@@ -216,10 +216,10 @@ class PgDropStat(object):
 
  counter_pg_drop_array = [ "SAI_INGRESS_PRIORITY_GROUP_STAT_DROPPED_PACKETS"]
  try:
- pickle.dump(self.get_counts_table(
+ json.dump(self.get_counts_table(
  counter_pg_drop_array,
  COUNTERS_PG_NAME_MAP),
- open(self.port_drop_stats_file, 'wb+'))
+ open(self.port_drop_stats_file, 'w+'))
  except IOError as e:
  print(e)
  sys.exit(e.errno)