Use the stdlib-list package to filter out Python standard library modules.

Author: thebjorn

pydeps/depgraph.py

@@ -7,8 +7,12 @@
 import pprint
 import re
 import enum
+import yaml
+
 from . import colors
 import sys
+import logging
+log = logging.getLogger(__name__)
 
 # we're normally not interested in imports of std python packages.
 PYLIB_PATH = {
@@ -40,6 +44,7 @@ def __init__(self, name, kind=imp.UNKNOWN, path=None, imports=(), exclude=False,
                 print "changing __main__ =>", self.name
         else:
             self.name = name
+
         self.kind = kind
         self.path = path             # needed here..?
         self.imports = set(imports)  # modules we import
@@ -58,12 +63,14 @@ def path_parts(self):
 
     @property
     def in_degree(self):
-        "Number of incoming arrows."
+        """Number of incoming arrows.
+        """
         return len(self.imports)
 
     @property
     def out_degree(self):
-        "Number of outgoing arrows."
+        """Number of outgoing arrows.
+        """
         return len(self.imported_by)
 
     @property
@@ -107,11 +114,18 @@ def __repr__(self):
         return json.dumps(self.__json__(), indent=4)
 
     def __iadd__(self, other):
+        if self.name == other.name and self.imports == other.imports and self.bacon == other.bacon:
+            return self
+        log.debug("iadd lhs: %r", self)
+        log.debug("iadd rhs: %r", other)
         assert self.name == other.name
         self.path = self.path or other.path
         self.kind = self.kind or other.kind
         self.imports |= other.imports
         self.imported_by |= other.imported_by
+        self.bacon = min(self.bacon, other.bacon)
+        self.excluded = self.excluded or other.excluded
+        log.debug("iadd result: %r", self)
         return self
 
     # def imported_modules(self, depgraph):
@@ -166,6 +180,7 @@ def get_colors(self, src, colorspace=None):
             return colorspace.color(src)
 
     def _is_pylib(self, path):
+        log.info('path %r in PYLIB_PATH %r => %s', path, PYLIB_PATH, path in PYLIB_PATH)
         return path in PYLIB_PATH
 
     def proximity_metric(self, a, b):
@@ -194,8 +209,8 @@ def dissimilarity_metric(self, a, b):
 
            Returns an int between 1 (default) and 4 (highly unrelated).
         """
-        if self._is_pylib(a) and self._is_pylib(b):
-            return 1
+        # if self._is_pylib(a) and self._is_pylib(b):
+        #     return 1
 
         res = 4
         for an, bn, n in izip_longest(a.name_parts, b.name_parts, range(4)):
@@ -205,6 +220,10 @@ def dissimilarity_metric(self, a, b):
         return res
 
     def _exclude(self, name):
+        # excl = any(skip.match(name) for skip in self.skiplist)
+        # if 'metar' in name:
+        #     print "Exclude?", name, excl
+        #     print [s.pattern for s in self.skiplist]
         return any(skip.match(name) for skip in self.skiplist)
 
     def __init__(self, depgraf, types, **args):
@@ -215,15 +234,17 @@ def __init__(self, depgraf, types, **args):
         self.cyclerelations = set()
 
         self.args = args
+
         self.sources = {}             # module_name -> Source
         self.skiplist = [re.compile(fnmatch.translate(arg)) for arg in args['exclude']]
+        # print "SKPLIST:", self.skiplist[0].pattern
 
         for name, imports in depgraf.items():
-            self.verbose(4, "depgraph:", name, imports)
+            log.debug("depgraph name=%r imports=%r", name, imports)
             src = Source(
                 name=name,
                 kind=imp(types.get(name, 0)),
-                imports=imports.keys(),
+                imports=imports.keys(),  # XXX: throwing away .values(), which is abspath!
                 args=args,
                 exclude=self._exclude(name),
             )
@@ -252,9 +273,11 @@ def __init__(self, depgraf, types, **args):
         self.exclude_bacon(self.args['max_bacon'])
 
         excluded = [v for v in self.sources.values() if v.excluded]
+        # print "EXCLUDED:", excluded
         self.skip_count = len(excluded)
         self.verbose(1, "skipping", self.skip_count, "modules")
         for module in excluded:
+            # print 'exclude:', module.name
             self.verbose(2, "  ", module.name)
 
         self.remove_excluded()
@@ -268,8 +291,10 @@ def verbose(self, n, *args):
 
     def add_source(self, src):
         if src.name in self.sources:
+            log.info("ADD-SOURCE[+=]\n%r", src)
             self.sources[src.name] += src
         else:
+            log.info("ADD-SOURCE[=]\n%r", src)
             self.sources[src.name] = src
 
     def __getitem__(self, item):
@@ -322,7 +347,8 @@ def traverse(node, path):
             traverse(src, [])
 
     def connect_generations(self):
-        "Traverse depth-first adding imported_by."
+        """Traverse depth-first adding imported_by.
+        """
         for src in self.sources.values():
             for _child in src.imports:
                 if _child in self.sources:
@@ -355,17 +381,22 @@ def exclude_noise(self):
             if src.is_noise():
                 self.verbose(2, "excluding", src, "because it is noisy:", src.degree)
                 src.excluded = True
+                print "Exluding noise:", src.name
                 self._add_skip(src.name)
 
     def exclude_bacon(self, limit):
-        "Exclude models that are more than `limit` hops away from __main__."
+        """Exclude models that are more than `limit` hops away from __main__.
+        """
         for src in self.sources.values():
             if src.bacon > limit:
                 src.excluded = True
+                # print "Excluding bacon:", src.name
                 self._add_skip(src.name)
 
     def remove_excluded(self):
-        "Remove all sources marked as excluded."
+        """Remove all sources marked as excluded.
+        """
+        # print yaml.dump({k:v.__json__() for k,v in self.sources.items()}, default_flow_style=False)
         sources = self.sources.values()
         for src in sources:
             if src.excluded:
@@ -374,4 +405,5 @@ def remove_excluded(self):
             src.imported_by = [m for m in src.imported_by if not self._exclude(m)]
 
     def _add_skip(self, name):
+        # print 'add skip:', name
         self.skiplist.append(re.compile(fnmatch.translate(name)))

pydeps/mf27.py

@@ -387,10 +387,13 @@ def scan_opcodes_25(self, co,
                 oparg_1, oparg_2, oparg_3 = unpack('<xHxHxH', code[:9])
                 level = consts[oparg_1]
                 if level == -1:  # normal import
+                    # print "import", (consts[oparg_2], names[oparg_3])
                     yield "import", (consts[oparg_2], names[oparg_3])
                 elif level == 0:  # absolute import
+                    # print "absolute_import", (consts[oparg_2], names[oparg_3])
                     yield "absolute_import", (consts[oparg_2], names[oparg_3])
                 else:  # relative import
+                    # print "relative_import", (level, consts[oparg_2], names[oparg_3])
                     yield "relative_import", (level, consts[oparg_2], names[oparg_3])
                 code = code[9:]
                 continue

pydeps/py2depgraph.py

@@ -19,12 +19,15 @@
 # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+import json
 import os
 import pprint
 import sys
 from collections import defaultdict
 
 import enum
+import yaml
+from .pystdlib import pystdlib
 
 from . import depgraph
 from . import mf27
@@ -83,6 +86,7 @@ def __init__(self, fname, *args, **kwargs):
         self.include_pylib_all = kwargs.pop('pylib_all', False)
 
         # include python std lib modules.
+        # self.include_pylib = kwargs.pop('pylib', self.include_pylib_all)
         self.include_pylib = kwargs.pop('pylib', self.include_pylib_all)
 
         self._depgraph = defaultdict(dict)
@@ -121,7 +125,8 @@ def _add_import(self, module):
                         pass
                     else:
                         rpath = os.path.split(module.__file__)[0].lower()
-                        pylib_p = [rpath.startswith(pp) for pp in PYLIB_PATH]
+                        if 'site-packages' not in rpath:
+                            pylib_p = [rpath.startswith(pp) for pp in PYLIB_PATH]
                     if self.include_pylib or not any(pylib_p):
                         # if self._last_caller.__name__ != module.__name__:
                         #     self._depgraph[self._last_caller.__name__][module.__name__] = module.__file__
@@ -288,23 +293,42 @@ def py2dep(pattern, **kw):
 
     # remove exclude so we don't pass it twice to modulefinder
     exclude = ['migrations'] + kw.pop('exclude', [])
+    log.debug("Exclude: %r", exclude)
+    log.debug("KW: %r", kw)
     mf = MyModuleFinder(path, exclude, **kw)
     mf.run_script(fname)
+    log.debug("mf._depgraph:\n%s", json.dumps(dict(mf._depgraph), indent=4))
 
     # remove dummy file and restore exclude argument
     os.unlink(fname)
     kw['exclude'] = exclude
 
-    if kw.get('verbose', 0) >= 4:  # pragma: nocover
-        print
-        print "mf27._depgraph:", mf._depgraph
-        print "mf27._types:   ", mf._types
-        print "mf27.modules:  ", pprint.pformat(mf.modules)
-        print
-        print "last caller:           ", mf._last_caller
-
-    log.debug("Returning depgraph.")
-    return depgraph.DepGraph(mf._depgraph, mf._types, **kw)
+    if kw.get('pylib'):
+        mf_depgraph = mf._depgraph
+        for k, v in mf._depgraph.items():
+            log.debug('depgraph item: %r %r', k, v)
+        # mf_modules = {k: os.path.abspath(v.__file__)
+        #               for k, v in mf.modules.items()}
+    else:
+        pylib = pystdlib()
+        mf_depgraph = {}
+        for k, v in mf._depgraph.items():
+            log.debug('depgraph item: %r %r', k, v)
+            if k in pylib:
+                continue
+            vals = {vk: vv for vk, vv in v.items() if vk not in pylib}
+            mf_depgraph[k] = vals
+
+        # mf_modules = {k: os.path.abspath(v.__file__)
+        #               for k, v in mf.modules.items()
+        #               if k not in pylib}
+
+    log.debug("mf_depgraph:\n%s",
+              yaml.dump(dict(mf_depgraph), default_flow_style=False))
+    # log.error("mf._types:\n%s", yaml.dump(mf._types, default_flow_style=False))
+    # log.debug("mf_modules:\n%s", yaml.dump(mf_modules, default_flow_style=False))
+
+    return depgraph.DepGraph(mf_depgraph, mf._types, **kw)
 
 
 def py2depgraph():

pydeps/pydeps.py

@@ -7,16 +7,13 @@
 import os
 import pprint
 import sys
+import textwrap
+
 from .py2depgraph import py2dep
 from .depgraph2dot import dep2dot, cycles2dot
 from .dot import dot
 from . import __version__
 import logging
-logging.basicConfig(
-    level=logging.CRITICAL,
-    # level=logging.DEBUG,
-    format='%(filename)s:%(lineno)d: %(levelname)s: %(message)s'
-)
 
 
 def _pydeps(**kw):
@@ -71,8 +68,21 @@ def parse_args(argv=()):
     _p.add_argument('--config', help="specify config file", metavar="FILE")
     _p.add_argument('--no-config', help="disable processing of config files", action='store_true')
     _p.add_argument('--version', action='store_true', help='print pydeps version')
+    _p.add_argument('-L', '--log', help=textwrap.dedent('''
+        set log-level to one of CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET.
+    '''))
     _args, argv = _p.parse_known_args(argv)
 
+    if _args.log:
+        loglevel = getattr(logging, _args.log)
+    else:
+        loglevel = None
+
+    logging.basicConfig(
+        level=loglevel,
+        format='%(filename)s:%(lineno)d: %(levelname)s: %(message)s'
+    )
+
     if _args.version:
         print "pydeps v" + __version__
         sys.exit(0)
@@ -136,7 +146,7 @@ def parse_args(argv=()):
     if _args.externals:
         return dict(
             T='svg', config=None, debug=False, display=None, exclude=[], externals=True,
-            fname=_args.fname, format='svg', max_bacon=2, no_config=False, nodot=False,
+            fname=_args.fname, format='svg', max_bacon=10, no_config=False, nodot=False,
             noise_level=200, noshow=True, output=None, pylib=False, pylib_all=False,
             show=False, show_cycles=False, show_deps=False, show_dot=False,
             show_raw_deps=False, verbose=0
@@ -173,8 +183,8 @@ def externals(pkgname):
     """
     kw = dict(
         T='svg', config=None, debug=False, display=None, exclude=[], externals=True,
-        format='svg', max_bacon=5, no_config=False, nodot=False,
-        noise_level=200, noshow=True, output=None, pylib=False, pylib_all=False,
+        format='svg', max_bacon=2**65, no_config=False, nodot=False,
+        noise_level=2**65, noshow=True, output=None, pylib=False, pylib_all=True,
         show=False, show_cycles=False, show_deps=False, show_dot=False,
         show_raw_deps=False, verbose=0
     )

pydeps/pystdlib.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+
+import sys
+import stdlib_list
+
+
+def pystdlib():
+    """Return a set of all module-names in the Python standard library.
+    """
+    curver = '.'.join(str(x) for x in sys.version_info[:2])
+    return (set(stdlib_list.stdlib_list(curver)) | {
+        '_LWPCookieJar', '_MozillaCookieJar', '_abcoll', 'email._parseaddr',
+        'email.base64mime', 'email.feedparser', 'email.quoprimime',
+        'encodings', 'genericpath', 'ntpath', 'nturl2path', 'os2emxpath',
+        'posixpath', 'sre_compile', 'sre_parse', 'unittest.case',
+        'unittest.loader', 'unittest.main', 'unittest.result',
+        'unittest.runner', 'unittest.signals', 'unittest.suite',
+        'unittest.util', '_threading_local', 'sre_constants', 'strop',
+        'repr', 'opcode', 'nt', 'encodings.aliases',
+        '_bisect', '_codecs', '_collections', '_functools', '_hashlib',
+        '_heapq', '_io', '_locale', '_LWPCookieJar', '_md5',
+        '_MozillaCookieJar', '_random', '_sha', '_sha256', '_sha512',
+        '_socket', '_sre', '_ssl', '_struct', '_subprocess',
+        '_threading_local', '_warnings', '_weakref', '_weakrefset',
+        '_winreg'
+    }) - {'__main__'}

requirements.txt

@@ -1,8 +1,9 @@
-PyYAML==3.11
-cov-core==1.15.0
-coverage==3.7.1
-enum34==1.0.4
-py==1.4.26
-pytest==2.6.4
-pytest-cov==1.8.1
-Sphinx==1.2.3
+PyYAML==3.12
+cov-core==1.15.0
+coverage==4.2
+enum34==1.0.4
+py==1.4.31
+pytest==3.0.3
+pytest-cov==2.4.0
+Sphinx==1.4.8
+stdlib-list>=0.3.3