commit 3ce9af43bac2fa293f1bf159585ab0e947652314
author CrazyBolillo <antonio@zoftko.com> Wed Apr 03 21:32:51 2024 -0600
committer Jacob Walls <jacobtylerwalls@gmail.com> Sat May 04 17:35:28 2024 -0400
tree 22f922b79c877c830e0202703179f0e16c8798c8
parent 47c0b8f50a486da2ce05702ee064079ddb0c6423

Improve performance by caching find_spec

Certain checkers upstream on pylint like import-error heavily use
find_spec. This method is IO intensive as it looks for files
across several search paths to return a ModuleSpec.

Since imports across files may repeat themselves it makes sense to cache
this method in order to speed up the linting process.

Closes pylint-dev/pylint#9310.

astroid/interpreter/_import/spec.py

diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py
index 93096e5..e1c5ed0 100644
--- a/astroid/interpreter/_import/spec.py
+++ b/astroid/interpreter/_import/spec.py
@@ -16,6 +16,7 @@
 import warnings
 import zipimport
 from collections.abc import Iterator, Sequence
+from functools import lru_cache
 from pathlib import Path
 from typing import Any, Literal, NamedTuple, Protocol
 
@@ -440,10 +441,15 @@
     :return: A module spec, which describes how the module was
              found and where.
     """
+    return _find_spec(tuple(modpath), tuple(path) if path else None)
+
+
+@lru_cache(maxsize=1024)
+def _find_spec(modpath: tuple, path: tuple) -> ModuleSpec:
     _path = path or sys.path
 
     # Need a copy for not mutating the argument.
-    modpath = modpath[:]
+    modpath = list(modpath)
 
     submodule_path = None
     module_parts = modpath[:]
@@ -468,3 +474,7 @@
             spec = spec._replace(submodule_search_locations=submodule_path)
 
     return spec
+
+
+def clear_spec_cache() -> None:
+    _find_spec.cache_clear()

astroid/manager.py

diff --git a/astroid/manager.py b/astroid/manager.py
index a7a51f1..195ac66 100644
--- a/astroid/manager.py
+++ b/astroid/manager.py
@@ -442,10 +442,12 @@
         # pylint: disable=import-outside-toplevel
         from astroid.brain.helpers import register_all_brains
         from astroid.inference_tip import clear_inference_tip_cache
+        from astroid.interpreter._import.spec import clear_spec_cache
         from astroid.interpreter.objectmodel import ObjectModel
         from astroid.nodes._base_nodes import LookupMixIn
         from astroid.nodes.scoped_nodes import ClassDef
 
+        clear_spec_cache()
         clear_inference_tip_cache()
         _invalidate_cache()  # inference context cache
 

tests/test_manager.py

diff --git a/tests/test_manager.py b/tests/test_manager.py
index 7861927..160fa94 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -23,6 +23,7 @@
     AttributeInferenceError,
 )
 from astroid.interpreter._import import util
+from astroid.interpreter._import.spec import clear_spec_cache
 from astroid.modutils import EXT_LIB_DIRS, module_in_path
 from astroid.nodes import Const
 from astroid.nodes.scoped_nodes import ClassDef, Module
@@ -41,6 +42,7 @@
 ):
     def setUp(self) -> None:
         super().setUp()
+        clear_spec_cache()
         self.manager = test_utils.brainless_manager()
 
     def test_ast_from_file(self) -> None:

tests/test_modutils.py

diff --git a/tests/test_modutils.py b/tests/test_modutils.py
index 929c589..be7095e 100644
--- a/tests/test_modutils.py
+++ b/tests/test_modutils.py
@@ -22,6 +22,7 @@
 from astroid import modutils
 from astroid.const import PY310_PLUS
 from astroid.interpreter._import import spec
+from astroid.interpreter._import.spec import clear_spec_cache
 
 from . import resources
 
@@ -41,6 +42,7 @@
     package = "mypypa"
 
     def tearDown(self) -> None:
+        clear_spec_cache()
         for k in list(sys.path_importer_cache):
             if "MyPyPa" in k:
                 del sys.path_importer_cache[k]