| # Copyright (c) 2006-2011, 2013-2014 LOGILAB S.A. (Paris, FRANCE) <contact@logilab.fr> |
| # Copyright (c) 2014-2016 Claudiu Popa <pcmanticore@gmail.com> |
| # Copyright (c) 2014 Google, Inc. |
| # Copyright (c) 2015-2016 Cara Vinson <ceridwenv@gmail.com> |
| |
| # Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html |
| # For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER |
| |
| """astroid manager: avoid multiple astroid build of a same module when |
| possible by providing a class responsible to get astroid representation |
| from various source and using a cache of built modules) |
| """ |
| |
| import os |
| import sys |
| import zipimport |
| |
| import six |
| |
| from astroid import exceptions |
| from astroid.interpreter._import import spec |
| from astroid import modutils |
| from astroid import transforms |
| from astroid import util |
| |
| builder = util.lazy_import('builder') |
| |
| def safe_repr(obj): |
| try: |
| return repr(obj) |
| except Exception: # pylint: disable=broad-except |
| return '???' |
| |
| |
| class AstroidManager(object): |
| """the astroid manager, responsible to build astroid from files |
| or modules. |
| |
| Use the Borg pattern. |
| """ |
| |
| name = 'astroid loader' |
| brain = {} |
| |
| def __init__(self): |
| self.__dict__ = AstroidManager.brain |
| if not self.__dict__: |
| # NOTE: cache entries are added by the [re]builder |
| self.astroid_cache = {} |
| self._mod_file_cache = {} |
| self._failed_import_hooks = [] |
| self.always_load_extensions = False |
| self.extension_package_whitelist = set() |
| self._transform = transforms.TransformVisitor() |
| |
| # Export these APIs for convenience |
| self.register_transform = self._transform.register_transform |
| self.unregister_transform = self._transform.unregister_transform |
| |
| def visit_transforms(self, node): |
| """Visit the transforms and apply them to the given *node*.""" |
| return self._transform.visit(node) |
| |
| def ast_from_file(self, filepath, modname=None, fallback=True, source=False): |
| """given a module name, return the astroid object""" |
| try: |
| filepath = modutils.get_source_file(filepath, include_no_ext=True) |
| source = True |
| except modutils.NoSourceFile: |
| pass |
| if modname is None: |
| try: |
| modname = '.'.join(modutils.modpath_from_file(filepath)) |
| except ImportError: |
| modname = filepath |
| if modname in self.astroid_cache and self.astroid_cache[modname].source_file == filepath: |
| return self.astroid_cache[modname] |
| if source: |
| return builder.AstroidBuilder(self).file_build(filepath, modname) |
| elif fallback and modname: |
| return self.ast_from_module_name(modname) |
| raise exceptions.AstroidBuildingError( |
| 'Unable to build an AST for {path}.', path=filepath) |
| |
| def _build_stub_module(self, modname): |
| return builder.AstroidBuilder(self).string_build('', modname) |
| |
| def _build_namespace_module(self, modname, path): |
| from astroid.builder import build_namespace_package_module |
| return build_namespace_package_module(modname, path) |
| |
| def _can_load_extension(self, modname): |
| if self.always_load_extensions: |
| return True |
| if modutils.is_standard_module(modname): |
| return True |
| parts = modname.split('.') |
| return any( |
| '.'.join(parts[:x]) in self.extension_package_whitelist |
| for x in range(1, len(parts) + 1)) |
| |
| def ast_from_module_name(self, modname, context_file=None): |
| """given a module name, return the astroid object""" |
| if modname in self.astroid_cache: |
| return self.astroid_cache[modname] |
| if modname == '__main__': |
| return self._build_stub_module(modname) |
| old_cwd = os.getcwd() |
| if context_file: |
| os.chdir(os.path.dirname(context_file)) |
| try: |
| found_spec = self.file_from_module_name(modname, context_file) |
| if found_spec.type == spec.ModuleType.PY_ZIPMODULE: |
| module = self.zip_import_data(found_spec.location) |
| if module is not None: |
| return module |
| |
| elif found_spec.type in (spec.ModuleType.C_BUILTIN, |
| spec.ModuleType.C_EXTENSION): |
| if (found_spec.type == spec.ModuleType.C_EXTENSION |
| and not self._can_load_extension(modname)): |
| return self._build_stub_module(modname) |
| try: |
| module = modutils.load_module_from_name(modname) |
| except Exception as ex: # pylint: disable=broad-except |
| util.reraise(exceptions.AstroidImportError( |
| 'Loading {modname} failed with:\n{error}', |
| modname=modname, path=spec.location, error=ex)) |
| return self.ast_from_module(module, modname) |
| |
| elif found_spec.type == spec.ModuleType.PY_COMPILED: |
| raise exceptions.AstroidImportError( |
| "Unable to load compiled module {modname}.", |
| modname=modname, path=found_spec.location) |
| |
| elif found_spec.type == spec.ModuleType.PY_NAMESPACE: |
| return self._build_namespace_module(modname, |
| found_spec.submodule_search_locations) |
| |
| if found_spec.location is None: |
| raise exceptions.AstroidImportError( |
| "Can't find a file for module {modname}.", |
| modname=modname) |
| |
| return self.ast_from_file(found_spec.location, modname, fallback=False) |
| except exceptions.AstroidBuildingError as e: |
| for hook in self._failed_import_hooks: |
| try: |
| return hook(modname) |
| except exceptions.AstroidBuildingError: |
| pass |
| raise e |
| finally: |
| os.chdir(old_cwd) |
| |
| def zip_import_data(self, filepath): |
| if zipimport is None: |
| return None |
| astroid_builder = builder.AstroidBuilder(self) |
| for ext in ('.zip', '.egg'): |
| try: |
| eggpath, resource = filepath.rsplit(ext + os.path.sep, 1) |
| except ValueError: |
| continue |
| try: |
| importer = zipimport.zipimporter(eggpath + ext) |
| zmodname = resource.replace(os.path.sep, '.') |
| if importer.is_package(resource): |
| zmodname = zmodname + '.__init__' |
| module = astroid_builder.string_build(importer.get_source(resource), |
| zmodname, filepath) |
| return module |
| except Exception: # pylint: disable=broad-except |
| continue |
| return None |
| |
| def file_from_module_name(self, modname, contextfile): |
| try: |
| value = self._mod_file_cache[(modname, contextfile)] |
| traceback = sys.exc_info()[2] |
| except KeyError: |
| try: |
| value = modutils.file_info_from_modpath( |
| modname.split('.'), context_file=contextfile) |
| traceback = sys.exc_info()[2] |
| except ImportError as ex: |
| value = exceptions.AstroidImportError( |
| 'Failed to import module {modname} with error:\n{error}.', |
| modname=modname, error=ex) |
| traceback = sys.exc_info()[2] |
| self._mod_file_cache[(modname, contextfile)] = value |
| if isinstance(value, exceptions.AstroidBuildingError): |
| six.reraise(exceptions.AstroidBuildingError, |
| value, traceback) |
| return value |
| |
| def ast_from_module(self, module, modname=None): |
| """given an imported module, return the astroid object""" |
| modname = modname or module.__name__ |
| if modname in self.astroid_cache: |
| return self.astroid_cache[modname] |
| try: |
| # some builtin modules don't have __file__ attribute |
| filepath = module.__file__ |
| if modutils.is_python_source(filepath): |
| return self.ast_from_file(filepath, modname) |
| except AttributeError: |
| pass |
| mock_ast = builder.AstroidBuilder(self).module_build(module, modname) |
| self.astroid_cache[modname] = mock_ast |
| return mock_ast |
| |
| def builtins(self): |
| """Get the builtins module |
| |
| This module is special since it's always built. |
| """ |
| return self.astroid_cache[six.moves.builtins.__name__] |
| |
| def register_failed_import_hook(self, hook): |
| """Registers a hook to resolve imports that cannot be found otherwise. |
| |
| `hook` must be a function that accepts a single argument `modname` which |
| contains the name of the module or package that could not be imported. |
| If `hook` can resolve the import, must return a node of type `astroid.Module`, |
| otherwise, it must raise `AstroidBuildingError`. |
| """ |
| self._failed_import_hooks.append(hook) |
| |
| def cache_module(self, module): |
| """Cache a module if no module with the same name is known yet.""" |
| self.astroid_cache.setdefault(module.name, module) |
| |
| def clear_cache(self): |
| self.astroid_cache.clear() |
| from astroid import raw_building |
| self.astroid_cache[six.moves.builtins.__name__] = raw_building.ast_from_builtins() |
