diff --git a/.gitignore b/.gitignore index 9a9c142a..b0c1d852 100644 --- a/.gitignore +++ b/.gitignore @@ -29,5 +29,9 @@ pip-log.txt # Doc build files _build +<<<<<<< HEAD # Visual Studio Code (vscode) config +======= +# VSCode config +>>>>>>> 333c0194e98e70e4f6a008f851f40f85089aed18 .vscode diff --git a/CHANGES.rst b/CHANGES.rst index f316d296..ed2b0e05 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,18 @@ Changes to mtools ================= +version 1.6.1 +~~~~~~~~~~~~~ + * mtools should use python3 in shebangs (#761) + * mloginfo: add timezone to mloginfo summary (#258) + * mloginfo --clients: more robust parsing of client metadata (#765) + * mloginfo --queries: fix missing patterns for remove command (#742) + * mloginfo --queries: add rounding option (#698) + * mplotqueries: fix error parsing checkpoint log line (#757) + +Thanks to @stennie, @kallimachos, @kevinadi, @niccottrell, and @p-mongo +for contributions to this release. + version 1.6.0 ~~~~~~~~~~~~~ Now supporting Python 3.6+ (only) diff --git a/doc/mloginfo.rst b/doc/mloginfo.rst index 927e21ad..21727e13 100644 --- a/doc/mloginfo.rst +++ b/doc/mloginfo.rst @@ -13,13 +13,21 @@ Usage .. code-block:: bash - mloginfo [-h] [--version] logfile - [--verbose] - [--queries] [--restarts] [--distinct] [--connections] [--rsstate] + mloginfo [-h] logfile [--clients] + [--connections] [--cursors] + [--distinct] + [--queries] + [--rounding {0,1,2,3,4}] + [--sort {namespace,pattern,count,min,max,mean,95%,sum}] + [--restarts] + [--rsstate] [--storagestats] - [--transactions] [--tsort {duration}] + [--transactions] + [--tsort {duration}] + [--verbose] + [--version] General Parameters ~~~~~~~~~~~~~~~~~~ @@ -130,6 +138,24 @@ In addition to the default information, this command will also output the serverside.auth_sessions update {"session_endtime": 1, "session_userid": 1} 1 244 244 244 0.2 244 False serverside.game_level find {"_id": 1} 1 104 104 104 0.1 104 None + +``--rounding`` +^^^^^^^^^^^^^^ + +This option adjusts the rounding for calculated statistics like mean and +95%-ile. + +For example: + +.. code-block:: bash + + mloginfo mongod.log --queries --rounding 2 + +This option has no effect unless ``--queries`` is also specified. + +Valid rounding values are from 0 to 4 decimal places. The default value is 1. + + ``--sort`` ^^^^^^^^^^ @@ -143,6 +169,11 @@ example: This option has no effect unless ``--queries`` is also specified. +Valid sort options are: ``namespace``, ``pattern``, ``count``, ``min``, +``max``, ``mean``, ``95%``, and ``sum``. + +The default sort option is ``sum``. + Restarts (``--restarts``) ------------------------- diff --git a/mtools/__init__.py b/mtools/__init__.py index 1ba3ccec..810e637c 100644 --- a/mtools/__init__.py +++ b/mtools/__init__.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 """Mtools init.""" __author__ = 'thomas@rueckstiess.net' diff --git a/mtools/mgenerate/mgenerate.py b/mtools/mgenerate/mgenerate.py index 356cc8d2..e510cb4a 100644 --- a/mtools/mgenerate/mgenerate.py +++ b/mtools/mgenerate/mgenerate.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ mgenerate is no longer included with mtools. Please use `mgeneratejs `__ instead. diff --git a/mtools/mlaunch/mlaunch.py b/mtools/mlaunch/mlaunch.py index ed81b9e2..101f91e8 100755 --- a/mtools/mlaunch/mlaunch.py +++ b/mtools/mlaunch/mlaunch.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function diff --git a/mtools/mlogfilter/mlogfilter.py b/mtools/mlogfilter/mlogfilter.py index c797aa79..1317554d 100644 --- a/mtools/mlogfilter/mlogfilter.py +++ b/mtools/mlogfilter/mlogfilter.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import inspect import re diff --git a/mtools/mloginfo/mloginfo.py b/mtools/mloginfo/mloginfo.py index cdabe837..34282227 100644 --- a/mtools/mloginfo/mloginfo.py +++ b/mtools/mloginfo/mloginfo.py @@ -1,5 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +import datetime import inspect import sys @@ -64,8 +65,15 @@ def run(self, arguments=None): print(" start: %s" % (start_time)) print(" end: %s" % (end_time)) - # TODO: add timezone if iso8601 format print("date format: %s" % self.logfile.datetime_format) + + # self.logfile.timezone is a dateutil.tzinfo object + tzdt = datetime.datetime.now(self.logfile.timezone) + if (tzdt.tzname()): + timezone = tzdt.tzname() + else: + timezone = f"UTC {tzdt.strftime('%z')}" + print(" timezone: %s" % timezone) print(" length: %s" % len(self.logfile)) print(" binary: %s" % (self.logfile.binary or "unknown")) diff --git a/mtools/mloginfo/sections/clients_section.py b/mtools/mloginfo/sections/clients_section.py index d0b981f6..568dba3c 100644 --- a/mtools/mloginfo/sections/clients_section.py +++ b/mtools/mloginfo/sections/clients_section.py @@ -54,24 +54,25 @@ def run(self): ip_formatted = str(ip) if ip_formatted != "127.0.0.1": - # driver metadata is not strict JSON, parsing string - # then adding required double quotes to keys - driver_data_raw = ("{" + line.split("{", 1)[1].split("}")[0] + "}}") - driver_data = (re.sub(r"(\w+): ", r'"\1":', driver_data_raw)) - driver_data_json = json.loads(driver_data) - - driver = driver_data_json["driver"]["name"] - version = driver_data_json["driver"]["version"] - dv_formatted = str(driver) + ":" + str(version) - if dv_formatted not in driver_info: - driver_info[dv_formatted] = [ip_formatted] - elif dv_formatted in driver_info: - if ip_formatted in driver_info.get(dv_formatted): - continue - else: - driver_info[dv_formatted].append(ip_formatted) + client_metadata = logevent.client_metadata + try: + driver = client_metadata["driver"]["name"] + version = client_metadata["driver"]["version"] + dv_formatted = str(driver) + ":" + str(version) + if dv_formatted not in driver_info: + driver_info[dv_formatted] = [ip_formatted] + elif dv_formatted in driver_info: + if ip_formatted in driver_info.get(dv_formatted): + continue + else: + driver_info[dv_formatted].append(ip_formatted) + except Exception: + next print('%-15s - Unique connections' % 'Driver:Version ') - for key, value in sorted(driver_info.items()): - print("%-15s : " % (key) + str(value)) + if len(driver_info): + for key, value in sorted(driver_info.items()): + print("%-15s : " % (key) + str(value)) + else: + print("No client metadata found") print('') diff --git a/mtools/mloginfo/sections/query_section.py b/mtools/mloginfo/sections/query_section.py index 8dde976a..360f58fa 100644 --- a/mtools/mloginfo/sections/query_section.py +++ b/mtools/mloginfo/sections/query_section.py @@ -43,6 +43,13 @@ def __init__(self, mloginfo): 'mean', '95%', 'sum']) + helptext = 'Number of decimal places for rounding of calculated stats' + self.mloginfo.argparser_sectiongroup.add_argument('--rounding', + action='store', + type=int, + default=1, + choices=range(0, 5), + help=helptext) @property def active(self): @@ -54,6 +61,7 @@ def run(self): grouping = Grouping(group_by=lambda x: (x.namespace, x.operation, x.pattern, x.allowDiskUse)) logfile = self.mloginfo.logfile + rounding = self.mloginfo.args['rounding'] if logfile.start and logfile.end: progress_start = self.mloginfo._datetime_to_epoch(logfile.start) @@ -113,14 +121,13 @@ def run(self): stats['count'] = len(group_events_all) stats['min'] = min(group_events) if group_events else 0 stats['max'] = max(group_events) if group_events else 0 - stats['mean'] = 0 if np: - stats['95%'] = (np.percentile(group_events, 95) + stats['95%'] = (round(np.percentile(group_events, 95), rounding) if group_events else 0) else: stats['95%'] = 'n/a' stats['sum'] = sum(group_events) if group_events else 0 - stats['mean'] = (stats['sum'] / stats['count'] + stats['mean'] = (round(stats['sum'] / stats['count'], rounding) if group_events else 0) if self.mloginfo.args['verbose']: diff --git a/mtools/mlogvis/mlogvis.py b/mtools/mlogvis/mlogvis.py index 3d3ce6a6..16c18faa 100644 --- a/mtools/mlogvis/mlogvis.py +++ b/mtools/mlogvis/mlogvis.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os import sys diff --git a/mtools/mplotqueries/mplotqueries.py b/mtools/mplotqueries/mplotqueries.py index 92f412b0..f0333c35 100644 --- a/mtools/mplotqueries/mplotqueries.py +++ b/mtools/mplotqueries/mplotqueries.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from six.moves import cPickle import glob diff --git a/mtools/util/__init__.py b/mtools/util/__init__.py index 6f1a59ef..e82198cc 100644 --- a/mtools/util/__init__.py +++ b/mtools/util/__init__.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 """Make OrderedDict available: collections (py2.7) or ordereddict (py2.6).""" try: diff --git a/mtools/util/cmdlinetool.py b/mtools/util/cmdlinetool.py index d4214a76..4c8a94a1 100644 --- a/mtools/util/cmdlinetool.py +++ b/mtools/util/cmdlinetool.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 """Command line tool utility.""" import argparse diff --git a/mtools/util/grouping.py b/mtools/util/grouping.py index 5011ecb6..382d5f98 100644 --- a/mtools/util/grouping.py +++ b/mtools/util/grouping.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 """Utility for grouping items and working with grouped items.""" import re diff --git a/mtools/util/hci.py b/mtools/util/hci.py index 807dcfef..492778ae 100644 --- a/mtools/util/hci.py +++ b/mtools/util/hci.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 """Utility for working with datetimes.""" import re diff --git a/mtools/util/input_source.py b/mtools/util/input_source.py index 779d6978..10dfae8b 100644 --- a/mtools/util/input_source.py +++ b/mtools/util/input_source.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 """Input source utility.""" diff --git a/mtools/util/log2code.py b/mtools/util/log2code.py index 3f5cc6b3..383b6f25 100644 --- a/mtools/util/log2code.py +++ b/mtools/util/log2code.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 from __future__ import print_function diff --git a/mtools/util/logcodeline.py b/mtools/util/logcodeline.py index e9575141..9b2de1a2 100644 --- a/mtools/util/logcodeline.py +++ b/mtools/util/logcodeline.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 from collections import defaultdict diff --git a/mtools/util/logevent.py b/mtools/util/logevent.py index 457ceb74..9dc9c7b1 100644 --- a/mtools/util/logevent.py +++ b/mtools/util/logevent.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 import json import re @@ -119,7 +119,6 @@ def _reset(self): self._counters_calculated = False self._allowDiskUse = None - self._bytesRead = None self._bytesWritten = None self._timeReadingMicros = None @@ -151,6 +150,9 @@ def _reset(self): self._component = None self.merge_marker_str = '' + self._client_metadata_calculated = False + self._client_metadata = None + def set_line_str(self, line_str): """ Set line_str. @@ -215,7 +217,7 @@ def duration(self): if matchobj: self._duration = int(matchobj.group(1)) # SERVER-16176 - Logging of slow checkpoints - elif "Checkpoint" in self.line_str: + elif "Checkpoint took" in self.line_str: matchobj = re.search("Checkpoint took ([\d]+) seconds to complete", self.line_str) if matchobj: self._duration = int(matchobj.group(1)) * 1000 @@ -938,6 +940,29 @@ def _extract_level(self): self._level = False self._component = False + @property + def client_metadata(self): + """Return client metadata.""" + if not self._client_metadata_calculated: + self._client_metadata_calculated = True + + line_str = self.line_str + if (line_str and line_str.find('client metadata')): + try: + metadata_pos = line_str.find("{") + if metadata_pos == -1: + return + else: + metadata = line_str[metadata_pos:] + # Make valid JSON by wrapping field names in quotes + metadata, _ = re.subn(r'([{,])\s*([^,{\s\'"]+)\s*:', + ' \\1 "\\2" : ', metadata) + self._client_metadata = json.loads(metadata) + except ValueError: + self._client_metadata = None + + return self._client_metadata + def parse_all(self): """ Trigger extraction of all information. diff --git a/mtools/util/logfile.py b/mtools/util/logfile.py index 5b1a1698..8b2f2738 100644 --- a/mtools/util/logfile.py +++ b/mtools/util/logfile.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 from __future__ import print_function diff --git a/mtools/util/parse_sourcecode.py b/mtools/util/parse_sourcecode.py index 041f40d6..724955d8 100755 --- a/mtools/util/parse_sourcecode.py +++ b/mtools/util/parse_sourcecode.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from six.moves import cPickle import os diff --git a/mtools/util/pattern.py b/mtools/util/pattern.py index dfe1bd86..c52a6ffd 100644 --- a/mtools/util/pattern.py +++ b/mtools/util/pattern.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 import json import re diff --git a/mtools/util/presplit.py b/mtools/util/presplit.py index 17d874be..36ed98c0 100644 --- a/mtools/util/presplit.py +++ b/mtools/util/presplit.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 from __future__ import print_function diff --git a/mtools/util/print_table.py b/mtools/util/print_table.py index 57ea77b7..e6b3a34a 100644 --- a/mtools/util/print_table.py +++ b/mtools/util/print_table.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 def print_table(rows, override_headers=None, uppercase_headers=True): """All rows need to be a list of dictionaries, all with the same keys.""" diff --git a/mtools/util/profile_collection.py b/mtools/util/profile_collection.py index d981b16a..32279af9 100644 --- a/mtools/util/profile_collection.py +++ b/mtools/util/profile_collection.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 from dateutil.tz import tzutc from pymongo import ASCENDING, DESCENDING diff --git a/mtools/version.py b/mtools/version.py index 24d745f4..42be23dd 100644 --- a/mtools/version.py +++ b/mtools/version.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 """Mtools version.""" -__version__ = '1.6.1-dev' +__version__ = '1.6.2-dev' diff --git a/setup.py b/setup.py index f0ad4c7a..220ec2d2 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/env python3 """Setup file for mtools.""" import platform