Decrease the time required to import astroid

[Pierre-Sassoulas]

Steps to reproduce

python3 -X importtime -c 'import astroid'

Current behavior

It takes a half second to import astroid. This account for 92% of pylint's startup time in 2.13.0-dev0. See pylint-dev/pylint#4814 for details.

import time: self [us] | cumulative | imported package
import time:       182 |        182 |   _io
import time:        59 |         59 |   marshal
import time:       213 |        213 |   posix
import time:       463 |        916 | _frozen_importlib_external
import time:       143 |        143 |   time
import time:       210 |        353 | zipimport
import time:        68 |         68 |     _codecs
import time:      8157 |       8225 |   codecs
import time:      4532 |       4532 |   encodings.aliases
import time:      2092 |      14848 | encodings
import time:       526 |        526 | encodings.utf_8
import time:       183 |        183 | _signal
import time:       910 |        910 | encodings.latin_1
import time:        34 |         34 |     _abc
import time:       527 |        560 |   abc
import time:       539 |       1099 | io
import time:        43 |         43 |       _stat
import time:       522 |        564 |     stat
import time:      1066 |       1066 |     _collections_abc
import time:       459 |        459 |       genericpath
import time:       481 |        939 |     posixpath
import time:       781 |       3349 |   os
import time:       451 |        451 |   _sitebuiltins
import time:        57 |         57 |     _locale
import time:       411 |        467 |   _bootlocale
import time:      1220 |       1220 |     apport_python_hook
import time:       396 |       1615 |   sitecustomize
import time:      4927 |      10808 | site
import time:       989 |        989 |     types
import time:       833 |        833 |     warnings
import time:      1833 |       3654 |   importlib
import time:       954 |        954 |         enum
import time:        58 |         58 |           _sre
import time:       560 |        560 |             sre_constants
import time:      1033 |       1593 |           sre_parse
import time:       788 |       2437 |         sre_compile
import time:        58 |         58 |               _operator
import time:      1129 |       1187 |             operator
import time:       424 |        424 |             keyword
import time:        34 |         34 |               _heapq
import time:       499 |        533 |             heapq
import time:        85 |         85 |             itertools
import time:       496 |        496 |             reprlib
import time:        47 |         47 |             _collections
import time:      1360 |       4129 |           collections
import time:        37 |         37 |           _functools
import time:      3949 |       8114 |         functools
import time:       451 |        451 |         copyreg
import time:       751 |      12704 |       re
import time:       547 |      13251 |     fnmatch
import time:        70 |         70 |       nt
import time:        49 |         49 |       nt
import time:        46 |         46 |       nt
import time:        46 |         46 |       nt
import time:       608 |        817 |     ntpath
import time:        47 |         47 |     errno
import time:       734 |        734 |       urllib
import time:      1519 |       2253 |     urllib.parse
import time:      3700 |      20066 |   pathlib
import time:      1277 |       1277 |   astroid.__pkginfo__
import time:       423 |        423 |         collections.abc
import time:       704 |        704 |         contextlib
import time:      1683 |       2809 |       typing
import time:       127 |        127 |               _opcode
import time:       619 |        745 |             opcode
import time:       744 |       1488 |           dis
import time:       442 |        442 |           importlib.machinery
import time:       480 |        480 |               token
import time:      1048 |       1527 |             tokenize
import time:       389 |       1915 |           linecache
import time:      1941 |       5784 |         inspect
import time:       518 |        518 |               _weakrefset
import time:       873 |       1390 |             weakref
import time:      1384 |       1384 |             wrapt._wrappers
import time:      6921 |       9695 |           wrapt.wrappers
import time:      1350 |       1350 |             threading
import time:      1461 |       2811 |           wrapt.decorators
import time:      1089 |       1089 |           wrapt.importer
import time:      2134 |      15727 |         wrapt
import time:        77 |         77 |             copy_reg
import time:       449 |        449 |               lazy_object_proxy.utils_py3
import time:       550 |        999 |             lazy_object_proxy.utils
import time:      1466 |       1466 |             lazy_object_proxy.cext
import time:       572 |        572 |             lazy_object_proxy._version
import time:      4391 |       7503 |           lazy_object_proxy
import time:       687 |       8190 |         astroid.util
import time:       635 |        635 |           pprint
import time:      2008 |       2643 |         astroid.context
import time:      1714 |       1714 |         astroid.exceptions
import time:      7641 |       7641 |         typing_extensions
import time:       616 |      42312 |       astroid.decorators
import time:       594 |        594 |       astroid.mixins
import time:      1371 |       1371 |         astroid.const
import time:      1881 |       1881 |           astroid.interpreter._import
import time:      1275 |       1275 |             distutils
import time:       385 |        385 |                 __future__
import time:        68 |         68 |                   binascii
import time:       631 |        631 |                     importlib.abc
import time:       503 |       1133 |                   importlib.util
import time:        66 |         66 |                     zlib
import time:       367 |        367 |                       _compression
import time:       251 |        251 |                       _bz2
import time:       485 |       1103 |                     bz2
import time:       217 |        217 |                       _lzma
import time:       431 |        647 |                     lzma
import time:        36 |         36 |                     pwd
import time:        26 |         26 |                     grp
import time:      1033 |       2908 |                   shutil
import time:        66 |         66 |                     _struct
import time:       337 |        402 |                   struct
import time:      1019 |       5528 |                 zipfile
import time:       599 |        599 |                 pkgutil
import time:      1913 |       1913 |                 platform
import time:        53 |         53 |                     math
import time:       120 |        120 |                     _datetime
import time:      1114 |       1286 |                   datetime
import time:      1129 |       1129 |                       xml
import time:      1078 |       2206 |                     xml.parsers
import time:       107 |        107 |                     pyexpat
import time:       431 |       2744 |                   xml.parsers.expat
import time:      1234 |       5263 |                 plistlib
import time:       578 |        578 |                   email
import time:       674 |        674 |                     email.errors
import time:        36 |         36 |                             _string
import time:       843 |        878 |                           string
import time:       465 |       1343 |                         email.quoprimime
import time:       524 |        524 |                           base64
import time:       402 |        925 |                         email.base64mime
import time:       451 |        451 |                             quopri
import time:       418 |        869 |                           email.encoders
import time:       874 |       1743 |                         email.charset
import time:       950 |       4960 |                       email.header
import time:        34 |         34 |                             _bisect
import time:       424 |        457 |                           bisect
import time:        27 |         27 |                           _sha512
import time:        22 |         22 |                           _random
import time:       616 |       1122 |                         random
import time:       136 |        136 |                           _socket
import time:       617 |        617 |                             select
import time:       885 |       1501 |                           selectors
import time:      1593 |       3229 |                         socket
import time:      1011 |       1011 |                             locale
import time:       919 |       1929 |                           calendar
import time:       557 |       2485 |                         email._parseaddr
import time:       813 |       7647 |                       email.utils
import time:       561 |      13167 |                     email._policybase
import time:       945 |      14785 |                   email.feedparser
import time:      8210 |      23572 |                 email.parser
import time:       721 |        721 |                 tempfile
import time:      1194 |       1194 |                 textwrap
import time:      6297 |       6297 |                 pkg_resources.extern
import time:      1130 |       1130 |                     pkg_resources._vendor
import time:      1168 |       2297 |                   pkg_resources._vendor.six
import time:       149 |       2446 |                 pkg_resources.extern.six
import time:       560 |        560 |                     pkg_resources._vendor.six
import time:        70 |        629 |                   pkg_resources._vendor.six.moves
import time:        56 |        685 |                 pkg_resources.extern.six.moves
import time:        20 |         20 |                   pkg_resources._vendor.six.moves
import time:        50 |         69 |                 pkg_resources._vendor.six.moves.urllib
import time:       461 |        461 |                 pkg_resources.py31compat
import time:       910 |        910 |                   pkg_resources._vendor.appdirs
import time:        81 |        990 |                 pkg_resources.extern.appdirs
import time:       436 |        436 |                     pkg_resources._vendor.packaging.__about__
import time:      3589 |       4025 |                   pkg_resources._vendor.packaging
import time:        72 |       4097 |                 pkg_resources.extern.packaging
import time:       378 |        378 |                   pkg_resources.extern.packaging._structures
import time:      2111 |       2489 |                 pkg_resources.extern.packaging.version
import time:       379 |        379 |                   pkg_resources.extern.packaging._compat
import time:      6610 |       6989 |                 pkg_resources.extern.packaging.specifiers
import time:        74 |         74 |                             org
import time:        17 |         90 |                           org.python
import time:        13 |        103 |                         org.python.core
import time:       539 |        641 |                       copy
import time:      1431 |       1431 |                       traceback
import time:     27210 |      29281 |                     pkg_resources._vendor.pyparsing
import time:       107 |      29388 |                   pkg_resources.extern.pyparsing
import time:        42 |         42 |                   pkg_resources.extern.six.moves.urllib
import time:      1463 |       1463 |                   pkg_resources.extern.packaging.markers
import time:      8461 |      39353 |                 pkg_resources.extern.packaging.requirements
import time:      4606 |       4606 |                 sysconfig
import time:     52302 |     159948 |               pkg_resources
import time:       338 |     160286 |             astroid.interpreter._import.util
import time:      1009 |     162569 |           astroid.interpreter._import.spec
import time:       614 |        614 |             distutils.errors
import time:       426 |        426 |                 distutils.dep_util
import time:       355 |        355 |                   distutils.debug
import time:       504 |        504 |                   distutils.log
import time:       548 |       1406 |                 distutils.spawn
import time:       559 |       2391 |               distutils.util
import time:       889 |       3280 |             distutils.sysconfig
import time:       854 |       4746 |           astroid.modutils
import time:       463 |        463 |           astroid.transforms
import time:      3030 |     172688 |         astroid.manager
import time:      4830 |     178888 |       astroid.bases
import time:      8974 |       8974 |       astroid.nodes.const
import time:       737 |        737 |         astroid.nodes.as_string
import time:      1306 |       2042 |       astroid.nodes.node_ng
import time:      5136 |     240752 |     astroid.nodes.node_classes
import time:       467 |        467 |       astroid.interpreter.dunder_lookup
import time:      1416 |       1883 |     astroid.nodes.scoped_nodes
import time:      4791 |     247424 |   astroid.nodes
import time:        57 |         57 |       _ast
import time:       764 |        820 |     ast
import time:    177912 |     177912 |       astroid.raw_building
import time:       656 |     178567 |     astroid.helpers
import time:       569 |        569 |       astroid.arguments
import time:       972 |       1541 |     astroid.protocols
import time:      1415 |     182342 |   astroid.inference
import time:      1575 |       1575 |   astroid.astroid_manager
import time:      1293 |       1293 |     astroid.brain
import time:       712 |       2005 |   astroid.brain.helpers
import time:        76 |         76 |           typed_ast
import time:        53 |        128 |         typed_ast.ast3
import time:       655 |        783 |       astroid._ast
import time:      1616 |       2399 |     astroid.rebuilder
import time:       561 |       2959 |   astroid.builder
import time:       355 |        355 |   astroid.inference_tip
import time:       701 |        701 |   astroid.objects
import time:       305 |        305 |   astroid.brain.brain_numpy_utils
import time:     46840 |     509498 | astroid

Expected behavior

Faster import, so pylint's can analyses empty file in less than half a second.

[Pierre-Sassoulas]

Doing #1103 would remove the import time from pkg_resources and distutils which are non negligible.

[Pierre-Sassoulas]

Another possible culprit is the use of NamedTuple, see https://lwn.net/Articles/730915/

[jacobtylerwalls]

Another possible culprit is the use of NamedTuple, see https://lwn.net/Articles/730915/

Thanks for the link, interesting read. Looks like it was optimized in Python 3.7.

[jacobtylerwalls]

By removing pkg_resources, #1536 reduces the time to import astroid to c.33% of what it was (i.e. 67% reduction) on a slow cloud console and about 50% of what it was on an M1 Mac. I think I will link that PR to close this issue.