From 724aec1fc5e1c30a7e150c147a06382aacf4445b Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 19 Nov 2019 16:41:06 -0600 Subject: [PATCH 1/2] introduce attrs and start discussion around stuff --- dvc/analytics.py | 279 ++++++++++++++++----------------------- dvc/config.py | 11 +- dvc/lock.py | 4 +- dvc/utils/serializers.py | 26 ++++ 4 files changed, 148 insertions(+), 172 deletions(-) create mode 100644 dvc/utils/serializers.py diff --git a/dvc/analytics.py b/dvc/analytics.py index 8b39fc31b8..9a9f69a484 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -1,4 +1,3 @@ -"""Collect and send usage analytics""" from __future__ import unicode_literals import errno @@ -6,251 +5,203 @@ import logging import os -from dvc import __version__ -from dvc.utils import env2bool +import attr + from dvc.utils.compat import str +from dvc.utils.serializers import json_serializer logger = logging.getLogger(__name__) -class Analytics(object): - """Class for collecting and sending usage analytics. - - Args: - info (dict): optional existing analytics report. - """ - - URL = "https://analytics.dvc.org" - TIMEOUT_POST = 5 - - USER_ID_FILE = "user_id" - - PARAM_DVC_VERSION = "dvc_version" - PARAM_USER_ID = "user_id" - PARAM_SYSTEM_INFO = "system_info" - - PARAM_OS = "os" +@attr.s +class SystemInfo: + linux_distro = attr.ib(default=None) + linux_distro_like = attr.ib(default=None) + linux_distro_version = attr.ib(default=None) + mac_version = attr.ib(default=None) + os = attr.ib(default=None) + windows_version_build = attr.ib(default=None) + windows_version_major = attr.ib(default=None) + windows_version_minor = attr.ib(default=None) + windows_version_service_pack = attr.ib(default=None) - PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major" - PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor" - PARAM_WINDOWS_VERSION_BUILD = "windows_version_build" - PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack" + def collect(self): + import platform - PARAM_MAC_VERSION = "mac_version" + system = platform.system().lower() + f = getattr(self, system) + f() + return self - PARAM_LINUX_DISTRO = "linux_distro" - PARAM_LINUX_DISTRO_VERSION = "linux_distro_version" - PARAM_LINUX_DISTRO_LIKE = "linux_distro_like" + def windows(self): + import sys - PARAM_SCM_CLASS = "scm_class" - PARAM_IS_BINARY = "is_binary" - PARAM_CMD_CLASS = "cmd_class" - PARAM_CMD_RETURN_CODE = "cmd_return_code" + version = sys.getwindowsversion() + self.os = "windows" + self.windows_version_major = version.major + self.windows_version_minor = version.minor + self.windows_version_build = version.build + self.windows_version_service_pack = version.service_pack - def __init__(self, info=None): - from dvc.config import Config - from dvc.lock import Lock + def darwin(self): + import platform - if info is None: - info = {} + self.os = "mac" + self.mac_version = platform.mac_ver()[0] - self.info = info + def linux(self): + import distro - cdir = Config.get_global_config_dir() - try: - os.makedirs(cdir) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise + self.os = "linux" + self.linux_distro = distro.id() + self.linux_distro_version = distro.version() + self.linux_distro_like = distro.like() - self.user_id_file = os.path.join(cdir, self.USER_ID_FILE) - self.user_id_file_lock = Lock(self.user_id_file + ".lock") - @staticmethod - def load(path): - """Loads analytics report from json file specified by path. +@json_serializer +@attr.s +class UserID: + import uuid + from dvc.config import Config - Args: - path (str): path to json file with analytics report. - """ - with open(path, "r") as fobj: - analytics = Analytics(info=json.load(fobj)) - os.unlink(path) - return analytics + user_id = attr.ib(default=attr.Factory(uuid.uuid4), converter=str) - def _write_user_id(self): - import uuid + config_dir = Config.get_global_config_dir() + fname = config_dir / "user_id" - with open(self.user_id_file, "w+") as fobj: - user_id = str(uuid.uuid4()) - info = {self.PARAM_USER_ID: user_id} - json.dump(info, fobj) - return user_id + @classmethod + def load(cls): + from dvc.lock import Lock, LockError + from json import JSONDecodeError - def _read_user_id(self): - if not os.path.exists(self.user_id_file): + if not cls.fname.exists: return None - with open(self.user_id_file, "r") as fobj: - try: - info = json.load(fobj) - except ValueError as exc: - logger.debug("Failed to load user_id: {}".format(exc)) - return None - - return info[self.PARAM_USER_ID] - - def _get_user_id(self): - from dvc.lock import LockError + lock = Lock(cls.fname.with_suffix(".lock")) try: - with self.user_id_file_lock: - user_id = self._read_user_id() - if user_id is None: - user_id = self._write_user_id() - return user_id - except LockError: - msg = "Failed to acquire '{}'" - logger.debug(msg.format(self.user_id_file_lock.lockfile)) + with lock: + return cls.from_file(cls.fname) - def _collect_windows(self): - import sys - - version = sys.getwindowsversion() # pylint: disable=no-member - info = {} - info[self.PARAM_OS] = "windows" - info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major - info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor - info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build - info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack - return info - - def _collect_darwin(self): - import platform - - info = {} - info[self.PARAM_OS] = "mac" - info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0] - return info + except ValueError as exc: + logger.debug("Failed to load user_id: {}".format(exc)) - def _collect_linux(self): - import distro + except JSONDecodeError: + logger.debug("Failed to read '{}'".format(fname)) - info = {} - info[self.PARAM_OS] = "linux" - info[self.PARAM_LINUX_DISTRO] = distro.id() - info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version() - info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like() - return info - - def _collect_system_info(self): - import platform + except LockError: + logger.debug("Failed to acquire '{}'".format(lock.lockfile)) - system = platform.system() + @classmethod + def generate(cls): + from dvc.utils import makedirs - if system == "Windows": - return self._collect_windows() + user_id = UserID() - if system == "Darwin": - return self._collect_darwin() + makedirs(cls.fname.parent, exist_ok=True) + user_id.to_file(cls.fname) + return user_id - if system == "Linux": - return self._collect_linux() - raise NotImplementedError +@json_serializer +@attr.s +class Report: + cmd_class = attr.ib(default=None) + cmd_return_code = attr.ib(default=None) + dvc_version = attr.ib(default=None) + is_binary = attr.ib(default=None) + scm_class = attr.ib(default=None) + user_id = attr.ib(default=None) + system_info = attr.ib(default=None) def collect(self): - """Collect analytics report.""" + from dvc import __version__ + from dvc.exceptions import NotDvcRepoError + from dvc.repo import Repo from dvc.scm import SCM from dvc.utils import is_binary - from dvc.repo import Repo - from dvc.exceptions import NotDvcRepoError - self.info[self.PARAM_DVC_VERSION] = __version__ - self.info[self.PARAM_IS_BINARY] = is_binary() - self.info[self.PARAM_USER_ID] = self._get_user_id() - - self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info() + self.dvc_version = __version__ + self.is_binary = is_binary() + self.user_id = (UserID.load() or UserID.generate()).user_id + self.system_info = SystemInfo().collect() try: scm = SCM(root_dir=Repo.find_root()) - self.info[self.PARAM_SCM_CLASS] = type(scm).__name__ + self.scm_class = type(scm).__name__ except NotDvcRepoError: pass def collect_cmd(self, args, ret): - """Collect analytics info from a CLI command.""" - from dvc.command.daemon import CmdDaemonAnalytics + if ret: + self.cmd_return_code = ret - assert isinstance(ret, int) or ret is None + if args and hasattr(args, "func"): + self.cmd_class = args.func.__name__ - if ret is not None: - self.info[self.PARAM_CMD_RETURN_CODE] = ret - if args is not None and hasattr(args, "func"): - assert args.func != CmdDaemonAnalytics - self.info[self.PARAM_CMD_CLASS] = args.func.__name__ +class Analytics(object): + def __init__(self, report=None): + self.report = report or Report() - def dump(self): - """Save analytics report to a temporary file. + @staticmethod + def load(path): + analytics = Analytics(report=Report.from_file(path)) + os.unlink(path) + return analytics - Returns: - str: path to the temporary file that contains the analytics report. - """ + def dump(self): import tempfile with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: - json.dump(self.info, fobj) + self.report.to_file(fobj.name) return fobj.name @staticmethod - def is_enabled(cmd=None): + def is_enabled(): from dvc.config import Config, to_bool - from dvc.command.daemon import CmdDaemonBase + from dvc.utils import env2bool if env2bool("DVC_TEST"): return False - if isinstance(cmd, CmdDaemonBase): - return False - core = Config(validate=False).config.get(Config.SECTION_CORE, {}) enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true")) + logger.debug( - "Analytics is {}.".format("enabled" if enabled else "disabled") + "Analytics is {status}." + .format(status="enabled" if enabled else "disabled") ) + return enabled @staticmethod def send_cmd(cmd, args, ret): - """Collect and send analytics for CLI command. - - Args: - args (list): parsed args for the CLI command. - ret (int): return value of the CLI command. - """ from dvc.daemon import daemon + from dvc.command.daemon import CmdDaemonBase - if not Analytics.is_enabled(cmd): - return + if not Analytics.is_enabled() or isinstance(cmd, CmdDaemonBase): + return False analytics = Analytics() - analytics.collect_cmd(args, ret) + analytics.report.collect_cmd(args, ret) daemon(["analytics", analytics.dump()]) def send(self): - """Collect and send analytics.""" import requests if not self.is_enabled(): return - self.collect() + self.report.collect() + + info = self.report.asdict + url = "https://analytics.dvc.org" - logger.debug("Sending analytics: {}".format(self.info)) + logger.debug("Sending analytics: {}".format(info)) try: - requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST) + requests.post(url, json=info, timeout=5) except requests.exceptions.RequestException as exc: logger.debug("Failed to send analytics: {}".format(str(exc))) diff --git a/dvc/config.py b/dvc/config.py index bd569864cb..c2d052a5f4 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -17,8 +17,7 @@ from dvc.exceptions import DvcException from dvc.exceptions import NotDvcRepoError -from dvc.utils.compat import open -from dvc.utils.compat import str +from dvc.utils.compat import open, str, pathlib logger = logging.getLogger(__name__) @@ -345,8 +344,8 @@ def get_global_config_dir(): """ from appdirs import user_config_dir - return user_config_dir( - appname=Config.APPNAME, appauthor=Config.APPAUTHOR + return pathlib.Path( + user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) ) @staticmethod @@ -358,8 +357,8 @@ def get_system_config_dir(): """ from appdirs import site_config_dir - return site_config_dir( - appname=Config.APPNAME, appauthor=Config.APPAUTHOR + return pathlib.Path( + site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) ) @staticmethod diff --git a/dvc/lock.py b/dvc/lock.py index 147118c0fe..dca1005e6e 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -58,7 +58,7 @@ def __init__(self, lockfile, tmp_dir=None): # [1] https://github.com/iterative/dvc/issues/2582 self._hostname = socket.gethostname() - self._lockfile = lockfile + self._lockfile = str(lockfile) self._lifetime = timedelta(days=365) # Lock for good by default self._separator = flufl.lock.SEP self._set_claimfile() @@ -111,7 +111,7 @@ class Lock(object): """ def __init__(self, lockfile, tmp_dir=None): - self.lockfile = lockfile + self.lockfile = str(lockfile) self._lock = None @property diff --git a/dvc/utils/serializers.py b/dvc/utils/serializers.py new file mode 100644 index 0000000000..03a48acdde --- /dev/null +++ b/dvc/utils/serializers.py @@ -0,0 +1,26 @@ +import abc +import json +import cattr + + +class JSONMixin(abc.ABC): + @classmethod + def from_file(cls, path): + with open(path, 'r') as fobj: + data = json.load(fobj) + + return cattr.structure(data, cls) + + def to_file(self, path): + with open(path, "w+") as fobj: + json.dump(cattr.unstructure(self), fobj) + + @property + def asdict(self): + return cattr.unstructure(self) + +def json_serializer(cls): + cls.from_file = classmethod(JSONMixin.from_file.__func__) + cls.to_file = JSONMixin.to_file + cls.asdict = JSONMixin.asdict + return cls From 5b2d4a5ef4a5ce814c5139f07e43035cc13aa1bd Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 20 Nov 2019 08:29:59 +0000 Subject: [PATCH 2/2] Restyled by black --- dvc/analytics.py | 33 +++++++++++++++++---------------- dvc/utils/serializers.py | 3 ++- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 9a9f69a484..96767d0629 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -16,14 +16,14 @@ @attr.s class SystemInfo: - linux_distro = attr.ib(default=None) - linux_distro_like = attr.ib(default=None) - linux_distro_version = attr.ib(default=None) - mac_version = attr.ib(default=None) - os = attr.ib(default=None) - windows_version_build = attr.ib(default=None) - windows_version_major = attr.ib(default=None) - windows_version_minor = attr.ib(default=None) + linux_distro = attr.ib(default=None) + linux_distro_like = attr.ib(default=None) + linux_distro_version = attr.ib(default=None) + mac_version = attr.ib(default=None) + os = attr.ib(default=None) + windows_version_build = attr.ib(default=None) + windows_version_major = attr.ib(default=None) + windows_version_minor = attr.ib(default=None) windows_version_service_pack = attr.ib(default=None) def collect(self): @@ -107,13 +107,13 @@ def generate(cls): @json_serializer @attr.s class Report: - cmd_class = attr.ib(default=None) + cmd_class = attr.ib(default=None) cmd_return_code = attr.ib(default=None) - dvc_version = attr.ib(default=None) - is_binary = attr.ib(default=None) - scm_class = attr.ib(default=None) - user_id = attr.ib(default=None) - system_info = attr.ib(default=None) + dvc_version = attr.ib(default=None) + is_binary = attr.ib(default=None) + scm_class = attr.ib(default=None) + user_id = attr.ib(default=None) + system_info = attr.ib(default=None) def collect(self): from dvc import __version__ @@ -170,8 +170,9 @@ def is_enabled(): enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true")) logger.debug( - "Analytics is {status}." - .format(status="enabled" if enabled else "disabled") + "Analytics is {status}.".format( + status="enabled" if enabled else "disabled" + ) ) return enabled diff --git a/dvc/utils/serializers.py b/dvc/utils/serializers.py index 03a48acdde..5aa8bad3d1 100644 --- a/dvc/utils/serializers.py +++ b/dvc/utils/serializers.py @@ -6,7 +6,7 @@ class JSONMixin(abc.ABC): @classmethod def from_file(cls, path): - with open(path, 'r') as fobj: + with open(path, "r") as fobj: data = json.load(fobj) return cattr.structure(data, cls) @@ -19,6 +19,7 @@ def to_file(self, path): def asdict(self): return cattr.unstructure(self) + def json_serializer(cls): cls.from_file = classmethod(JSONMixin.from_file.__func__) cls.to_file = JSONMixin.to_file