diff options
Diffstat (limited to 'src/main/python/venv/Lib/site-packages/astroid/modutils.py')
| -rw-r--r-- | src/main/python/venv/Lib/site-packages/astroid/modutils.py | 698 |
1 files changed, 698 insertions, 0 deletions
diff --git a/src/main/python/venv/Lib/site-packages/astroid/modutils.py b/src/main/python/venv/Lib/site-packages/astroid/modutils.py new file mode 100644 index 0000000..0c009b1 --- /dev/null +++ b/src/main/python/venv/Lib/site-packages/astroid/modutils.py @@ -0,0 +1,698 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2014-2018 Claudiu Popa <pcmanticore@gmail.com> +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2014 Denis Laxalde <denis.laxalde@logilab.fr> +# Copyright (c) 2014 LOGILAB S.A. (Paris, FRANCE) <contact@logilab.fr> +# Copyright (c) 2014 Eevee (Alex Munroe) <amunroe@yelp.com> +# Copyright (c) 2015 Florian Bruhin <me@the-compiler.org> +# Copyright (c) 2015 Radosław Ganczarek <radoslaw@ganczarek.in> +# Copyright (c) 2016 Derek Gustafson <degustaf@gmail.com> +# Copyright (c) 2016 Jakub Wilk <jwilk@jwilk.net> +# Copyright (c) 2016 Ceridwen <ceridwenv@gmail.com> +# Copyright (c) 2018 Mario Corchero <mcorcherojim@bloomberg.net> +# Copyright (c) 2018 Mario Corchero <mariocj89@gmail.com> +# Copyright (c) 2018 Anthony Sottile <asottile@umich.edu> + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Python modules manipulation utility functions. + +:type PY_SOURCE_EXTS: tuple(str) +:var PY_SOURCE_EXTS: list of possible python source file extension + +:type STD_LIB_DIRS: set of str +:var STD_LIB_DIRS: directories where standard modules are located + +:type BUILTIN_MODULES: dict +:var BUILTIN_MODULES: dictionary with builtin module names has key +""" +import imp +import os +import platform +import sys +import itertools +from distutils.sysconfig import get_python_lib # pylint: disable=import-error + +# pylint: disable=import-error, no-name-in-module +from distutils.errors import DistutilsPlatformError + +# distutils is replaced by virtualenv with a module that does +# weird path manipulations in order to get to the +# real distutils module. + +from .interpreter._import import spec +from .interpreter._import import util + +if sys.platform.startswith("win"): + PY_SOURCE_EXTS = ("py", "pyw") + PY_COMPILED_EXTS = ("dll", "pyd") +else: + PY_SOURCE_EXTS = ("py",) + PY_COMPILED_EXTS = ("so",) + + +try: + # The explicit sys.prefix is to work around a patch in virtualenv that + # replaces the 'real' sys.prefix (i.e. the location of the binary) + # with the prefix from which the virtualenv was created. This throws + # off the detection logic for standard library modules, thus the + # workaround. + STD_LIB_DIRS = { + get_python_lib(standard_lib=True, prefix=sys.prefix), + # Take care of installations where exec_prefix != prefix. + get_python_lib(standard_lib=True, prefix=sys.exec_prefix), + get_python_lib(standard_lib=True), + } +# get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to +# non-valid path, see https://bugs.pypy.org/issue1164 +except DistutilsPlatformError: + STD_LIB_DIRS = set() + +if os.name == "nt": + STD_LIB_DIRS.add(os.path.join(sys.prefix, "dlls")) + try: + # real_prefix is defined when running inside virtual environments, + # created with the **virtualenv** library. + STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "dlls")) + except AttributeError: + # sys.base_exec_prefix is always defined, but in a virtual environment + # created with the stdlib **venv** module, it points to the original + # installation, if the virtual env is activated. + try: + STD_LIB_DIRS.add(os.path.join(sys.base_exec_prefix, "dlls")) + except AttributeError: + pass + +if platform.python_implementation() == "PyPy": + _root = os.path.join(sys.prefix, "lib_pypy") + STD_LIB_DIRS.add(_root) + try: + # real_prefix is defined when running inside virtualenv. + STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "lib_pypy")) + except AttributeError: + pass + del _root +if os.name == "posix": + # Need the real prefix is we're under a virtualenv, otherwise + # the usual one will do. + try: + prefix = sys.real_prefix + except AttributeError: + prefix = sys.prefix + + def _posix_path(path): + base_python = "python%d.%d" % sys.version_info[:2] + return os.path.join(prefix, path, base_python) + + STD_LIB_DIRS.add(_posix_path("lib")) + if sys.maxsize > 2 ** 32: + # This tries to fix a problem with /usr/lib64 builds, + # where systems are running both 32-bit and 64-bit code + # on the same machine, which reflects into the places where + # standard library could be found. More details can be found + # here http://bugs.python.org/issue1294959. + # An easy reproducing case would be + # https://github.com/PyCQA/pylint/issues/712#issuecomment-163178753 + STD_LIB_DIRS.add(_posix_path("lib64")) + +EXT_LIB_DIRS = {get_python_lib(), get_python_lib(True)} +IS_JYTHON = platform.python_implementation() == "Jython" +BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True) + + +class NoSourceFile(Exception): + """exception raised when we are not able to get a python + source file for a precompiled file + """ + + +def _normalize_path(path): + return os.path.normcase(os.path.abspath(path)) + + +def _canonicalize_path(path): + return os.path.realpath(os.path.expanduser(path)) + + +def _path_from_filename(filename, is_jython=IS_JYTHON): + if not is_jython: + return filename + head, has_pyclass, _ = filename.partition("$py.class") + if has_pyclass: + return head + ".py" + return filename + + +def _handle_blacklist(blacklist, dirnames, filenames): + """remove files/directories in the black list + + dirnames/filenames are usually from os.walk + """ + for norecurs in blacklist: + if norecurs in dirnames: + dirnames.remove(norecurs) + elif norecurs in filenames: + filenames.remove(norecurs) + + +_NORM_PATH_CACHE = {} + + +def _cache_normalize_path(path): + """abspath with caching""" + # _module_file calls abspath on every path in sys.path every time it's + # called; on a larger codebase this easily adds up to half a second just + # assembling path components. This cache alleviates that. + try: + return _NORM_PATH_CACHE[path] + except KeyError: + if not path: # don't cache result for '' + return _normalize_path(path) + result = _NORM_PATH_CACHE[path] = _normalize_path(path) + return result + + +def load_module_from_name(dotted_name, path=None, use_sys=True): + """Load a Python module from its name. + + :type dotted_name: str + :param dotted_name: python name of a module or package + + :type path: list or None + :param path: + optional list of path where the module or package should be + searched (use sys.path if nothing or None is given) + + :type use_sys: bool + :param use_sys: + boolean indicating whether the sys.modules dictionary should be + used or not + + + :raise ImportError: if the module or package is not found + + :rtype: module + :return: the loaded module + """ + return load_module_from_modpath(dotted_name.split("."), path, use_sys) + + +def load_module_from_modpath(parts, path=None, use_sys=1): + """Load a python module from its split name. + + :type parts: list(str) or tuple(str) + :param parts: + python name of a module or package split on '.' + + :type path: list or None + :param path: + optional list of path where the module or package should be + searched (use sys.path if nothing or None is given) + + :type use_sys: bool + :param use_sys: + boolean indicating whether the sys.modules dictionary should be used or not + + :raise ImportError: if the module or package is not found + + :rtype: module + :return: the loaded module + """ + if use_sys: + try: + return sys.modules[".".join(parts)] + except KeyError: + pass + modpath = [] + prevmodule = None + for part in parts: + modpath.append(part) + curname = ".".join(modpath) + module = None + if len(modpath) != len(parts): + # even with use_sys=False, should try to get outer packages from sys.modules + module = sys.modules.get(curname) + elif use_sys: + # because it may have been indirectly loaded through a parent + module = sys.modules.get(curname) + if module is None: + mp_file, mp_filename, mp_desc = imp.find_module(part, path) + module = imp.load_module(curname, mp_file, mp_filename, mp_desc) + # mp_file still needs to be closed. + if mp_file: + mp_file.close() + if prevmodule: + setattr(prevmodule, part, module) + _file = getattr(module, "__file__", "") + prevmodule = module + if not _file and util.is_namespace(curname): + continue + if not _file and len(modpath) != len(parts): + raise ImportError("no module in %s" % ".".join(parts[len(modpath) :])) + path = [os.path.dirname(_file)] + return module + + +def load_module_from_file(filepath, path=None, use_sys=True, extrapath=None): + """Load a Python module from it's path. + + :type filepath: str + :param filepath: path to the python module or package + + :type path: list or None + :param path: + optional list of path where the module or package should be + searched (use sys.path if nothing or None is given) + + :type use_sys: bool + :param use_sys: + boolean indicating whether the sys.modules dictionary should be + used or not + + + :raise ImportError: if the module or package is not found + + :rtype: module + :return: the loaded module + """ + modpath = modpath_from_file(filepath, extrapath) + return load_module_from_modpath(modpath, path, use_sys) + + +def check_modpath_has_init(path, mod_path): + """check there are some __init__.py all along the way""" + modpath = [] + for part in mod_path: + modpath.append(part) + path = os.path.join(path, part) + if not _has_init(path): + old_namespace = util.is_namespace(".".join(modpath)) + if not old_namespace: + return False + return True + + +def _get_relative_base_path(filename, path_to_check): + """Extracts the relative mod path of the file to import from + + Check if a file is within the passed in path and if so, returns the + relative mod path from the one passed in. + + If the filename is no in path_to_check, returns None + + Note this function will look for both abs and realpath of the file, + this allows to find the relative base path even if the file is a + symlink of a file in the passed in path + + Examples: + _get_relative_base_path("/a/b/c/d.py", "/a/b") -> ["c","d"] + _get_relative_base_path("/a/b/c/d.py", "/dev") -> None + """ + importable_path = None + path_to_check = os.path.normcase(path_to_check) + abs_filename = os.path.abspath(filename) + if os.path.normcase(abs_filename).startswith(path_to_check): + importable_path = abs_filename + + real_filename = os.path.realpath(filename) + if os.path.normcase(real_filename).startswith(path_to_check): + importable_path = real_filename + + if importable_path: + base_path = os.path.splitext(importable_path)[0] + relative_base_path = base_path[len(path_to_check) :] + return [pkg for pkg in relative_base_path.split(os.sep) if pkg] + + return None + + +def modpath_from_file_with_callback(filename, extrapath=None, is_package_cb=None): + filename = os.path.expanduser(_path_from_filename(filename)) + + if extrapath is not None: + for path_ in itertools.chain(map(_canonicalize_path, extrapath), extrapath): + path = os.path.abspath(path_) + if not path: + continue + submodpath = _get_relative_base_path(filename, path) + if not submodpath: + continue + if is_package_cb(path, submodpath[:-1]): + return extrapath[path_].split(".") + submodpath + + for path in itertools.chain(map(_canonicalize_path, sys.path), sys.path): + path = _cache_normalize_path(path) + if not path: + continue + modpath = _get_relative_base_path(filename, path) + if not modpath: + continue + if is_package_cb(path, modpath[:-1]): + return modpath + + raise ImportError( + "Unable to find module for %s in %s" % (filename, ", \n".join(sys.path)) + ) + + +def modpath_from_file(filename, extrapath=None): + """given a file path return the corresponding split module's name + (i.e name of a module or package split on '.') + + :type filename: str + :param filename: file's path for which we want the module's name + + :type extrapath: dict + :param extrapath: + optional extra search path, with path as key and package name for the path + as value. This is usually useful to handle package split in multiple + directories using __path__ trick. + + + :raise ImportError: + if the corresponding module's name has not been found + + :rtype: list(str) + :return: the corresponding split module's name + """ + return modpath_from_file_with_callback(filename, extrapath, check_modpath_has_init) + + +def file_from_modpath(modpath, path=None, context_file=None): + return file_info_from_modpath(modpath, path, context_file).location + + +def file_info_from_modpath(modpath, path=None, context_file=None): + """given a mod path (i.e. split module / package name), return the + corresponding file, giving priority to source file over precompiled + file if it exists + + :type modpath: list or tuple + :param modpath: + split module's name (i.e name of a module or package split + on '.') + (this means explicit relative imports that start with dots have + empty strings in this list!) + + :type path: list or None + :param path: + optional list of path where the module or package should be + searched (use sys.path if nothing or None is given) + + :type context_file: str or None + :param context_file: + context file to consider, necessary if the identifier has been + introduced using a relative import unresolvable in the actual + context (i.e. modutils) + + :raise ImportError: if there is no such module in the directory + + :rtype: (str or None, import type) + :return: + the path to the module's file or None if it's an integrated + builtin module such as 'sys' + """ + if context_file is not None: + context = os.path.dirname(context_file) + else: + context = context_file + if modpath[0] == "xml": + # handle _xmlplus + try: + return _spec_from_modpath(["_xmlplus"] + modpath[1:], path, context) + except ImportError: + return _spec_from_modpath(modpath, path, context) + elif modpath == ["os", "path"]: + # FIXME: currently ignoring search_path... + return spec.ModuleSpec( + name="os.path", location=os.path.__file__, module_type=imp.PY_SOURCE + ) + return _spec_from_modpath(modpath, path, context) + + +def get_module_part(dotted_name, context_file=None): + """given a dotted name return the module part of the name : + + >>> get_module_part('astroid.as_string.dump') + 'astroid.as_string' + + :type dotted_name: str + :param dotted_name: full name of the identifier we are interested in + + :type context_file: str or None + :param context_file: + context file to consider, necessary if the identifier has been + introduced using a relative import unresolvable in the actual + context (i.e. modutils) + + + :raise ImportError: if there is no such module in the directory + + :rtype: str or None + :return: + the module part of the name or None if we have not been able at + all to import the given name + + XXX: deprecated, since it doesn't handle package precedence over module + (see #10066) + """ + # os.path trick + if dotted_name.startswith("os.path"): + return "os.path" + parts = dotted_name.split(".") + if context_file is not None: + # first check for builtin module which won't be considered latter + # in that case (path != None) + if parts[0] in BUILTIN_MODULES: + if len(parts) > 2: + raise ImportError(dotted_name) + return parts[0] + # don't use += or insert, we want a new list to be created ! + path = None + starti = 0 + if parts[0] == "": + assert ( + context_file is not None + ), "explicit relative import, but no context_file?" + path = [] # prevent resolving the import non-relatively + starti = 1 + while parts[starti] == "": # for all further dots: change context + starti += 1 + context_file = os.path.dirname(context_file) + for i in range(starti, len(parts)): + try: + file_from_modpath( + parts[starti : i + 1], path=path, context_file=context_file + ) + except ImportError: + if i < max(1, len(parts) - 2): + raise + return ".".join(parts[:i]) + return dotted_name + + +def get_module_files(src_directory, blacklist, list_all=False): + """given a package directory return a list of all available python + module's files in the package and its subpackages + + :type src_directory: str + :param src_directory: + path of the directory corresponding to the package + + :type blacklist: list or tuple + :param blacklist: iterable + list of files or directories to ignore. + + :type list_all: bool + :param list_all: + get files from all paths, including ones without __init__.py + + :rtype: list + :return: + the list of all available python module's files in the package and + its subpackages + """ + files = [] + for directory, dirnames, filenames in os.walk(src_directory): + if directory in blacklist: + continue + _handle_blacklist(blacklist, dirnames, filenames) + # check for __init__.py + if not list_all and "__init__.py" not in filenames: + dirnames[:] = () + continue + for filename in filenames: + if _is_python_file(filename): + src = os.path.join(directory, filename) + files.append(src) + return files + + +def get_source_file(filename, include_no_ext=False): + """given a python module's file name return the matching source file + name (the filename will be returned identically if it's already an + absolute path to a python source file...) + + :type filename: str + :param filename: python module's file name + + + :raise NoSourceFile: if no source file exists on the file system + + :rtype: str + :return: the absolute path of the source file if it exists + """ + filename = os.path.abspath(_path_from_filename(filename)) + base, orig_ext = os.path.splitext(filename) + for ext in PY_SOURCE_EXTS: + source_path = "%s.%s" % (base, ext) + if os.path.exists(source_path): + return source_path + if include_no_ext and not orig_ext and os.path.exists(base): + return base + raise NoSourceFile(filename) + + +def is_python_source(filename): + """ + rtype: bool + return: True if the filename is a python source file + """ + return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS + + +def is_standard_module(modname, std_path=None): + """try to guess if a module is a standard python module (by default, + see `std_path` parameter's description) + + :type modname: str + :param modname: name of the module we are interested in + + :type std_path: list(str) or tuple(str) + :param std_path: list of path considered has standard + + + :rtype: bool + :return: + true if the module: + - is located on the path listed in one of the directory in `std_path` + - is a built-in module + """ + modname = modname.split(".")[0] + try: + filename = file_from_modpath([modname]) + except ImportError: + # import failed, i'm probably not so wrong by supposing it's + # not standard... + return False + # modules which are not living in a file are considered standard + # (sys and __builtin__ for instance) + if filename is None: + # we assume there are no namespaces in stdlib + return not util.is_namespace(modname) + filename = _normalize_path(filename) + for path in EXT_LIB_DIRS: + if filename.startswith(_cache_normalize_path(path)): + return False + if std_path is None: + std_path = STD_LIB_DIRS + for path in std_path: + if filename.startswith(_cache_normalize_path(path)): + return True + return False + + +def is_relative(modname, from_file): + """return true if the given module name is relative to the given + file name + + :type modname: str + :param modname: name of the module we are interested in + + :type from_file: str + :param from_file: + path of the module from which modname has been imported + + :rtype: bool + :return: + true if the module has been imported relatively to `from_file` + """ + if not os.path.isdir(from_file): + from_file = os.path.dirname(from_file) + if from_file in sys.path: + return False + try: + stream, _, _ = imp.find_module(modname.split(".")[0], [from_file]) + + # Close the stream to avoid ResourceWarnings. + if stream: + stream.close() + return True + except ImportError: + return False + + +# internal only functions ##################################################### + + +def _spec_from_modpath(modpath, path=None, context=None): + """given a mod path (i.e. split module / package name), return the + corresponding spec + + this function is used internally, see `file_from_modpath`'s + documentation for more information + """ + assert modpath + location = None + if context is not None: + try: + found_spec = spec.find_spec(modpath, [context]) + location = found_spec.location + except ImportError: + found_spec = spec.find_spec(modpath, path) + location = found_spec.location + else: + found_spec = spec.find_spec(modpath, path) + if found_spec.type == spec.ModuleType.PY_COMPILED: + try: + location = get_source_file(found_spec.location) + return found_spec._replace( + location=location, type=spec.ModuleType.PY_SOURCE + ) + except NoSourceFile: + return found_spec._replace(location=location) + elif found_spec.type == spec.ModuleType.C_BUILTIN: + # integrated builtin module + return found_spec._replace(location=None) + elif found_spec.type == spec.ModuleType.PKG_DIRECTORY: + location = _has_init(found_spec.location) + return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE) + return found_spec + + +def _is_python_file(filename): + """return true if the given filename should be considered as a python file + + .pyc and .pyo are ignored + """ + return filename.endswith((".py", ".so", ".pyd", ".pyw")) + + +def _has_init(directory): + """if the given directory has a valid __init__ file, return its path, + else return None + """ + mod_or_pack = os.path.join(directory, "__init__") + for ext in PY_SOURCE_EXTS + ("pyc", "pyo"): + if os.path.exists(mod_or_pack + "." + ext): + return mod_or_pack + "." + ext + return None + + +def is_namespace(specobj): + return specobj.type == spec.ModuleType.PY_NAMESPACE + + +def is_directory(specobj): + return specobj.type == spec.ModuleType.PKG_DIRECTORY |