1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 """Python modules manipulation utility functions.
20
21 :type PY_SOURCE_EXTS: tuple(str)
22 :var PY_SOURCE_EXTS: list of possible python source file extension
23
24 :type STD_LIB_DIR: str
25 :var STD_LIB_DIR: directory where standard modules are located
26
27 :type BUILTIN_MODULES: dict
28 :var BUILTIN_MODULES: dictionary with builtin module names has key
29 """
30
31
32 __docformat__ = "restructuredtext en"
33
34 import sys
35 import os
36 from os.path import splitext, join, abspath, isdir, dirname, exists, basename
37 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
38 from distutils.sysconfig import get_config_var, get_python_lib, get_python_version
39 from distutils.errors import DistutilsPlatformError
40
41 try:
42 import zipimport
43 except ImportError:
44 zipimport = None
45
46 ZIPFILE = object()
47
48 from logilab.common import STD_BLACKLIST, _handle_blacklist
49
50
51
52
53
54
55
56 if sys.platform.startswith('win'):
57 PY_SOURCE_EXTS = ('py', 'pyw')
58 PY_COMPILED_EXTS = ('dll', 'pyd')
59 else:
60 PY_SOURCE_EXTS = ('py',)
61 PY_COMPILED_EXTS = ('so',)
62
63 try:
64 STD_LIB_DIR = get_python_lib(standard_lib=1)
65
66
67 except DistutilsPlatformError:
68 STD_LIB_DIR = '//'
69
70 EXT_LIB_DIR = get_python_lib()
71
72 BUILTIN_MODULES = dict(list(zip(sys.builtin_module_names,
73 [1]*len(sys.builtin_module_names))))
74
75
77 """exception raised when we are not able to get a python
78 source file for a precompiled file
79 """
80
83 self.module = module
84 self.obj = obj
85 self._imported = None
86
88 if self._imported is None:
89 self._imported = getattr(load_module_from_name(self.module),
90 self.obj)
91 return self._imported
92
94 try:
95 return super(LazyObject, self).__getattribute__(attr)
96 except AttributeError as ex:
97 return getattr(self._getobj(), attr)
98
100 return self._getobj()(*args, **kwargs)
101
102
104 """Load a Python module from its name.
105
106 :type dotted_name: str
107 :param dotted_name: python name of a module or package
108
109 :type path: list or None
110 :param path:
111 optional list of path where the module or package should be
112 searched (use sys.path if nothing or None is given)
113
114 :type use_sys: bool
115 :param use_sys:
116 boolean indicating whether the sys.modules dictionary should be
117 used or not
118
119
120 :raise ImportError: if the module or package is not found
121
122 :rtype: module
123 :return: the loaded module
124 """
125 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
126
127
129 """Load a python module from its splitted name.
130
131 :type parts: list(str) or tuple(str)
132 :param parts:
133 python name of a module or package splitted on '.'
134
135 :type path: list or None
136 :param path:
137 optional list of path where the module or package should be
138 searched (use sys.path if nothing or None is given)
139
140 :type use_sys: bool
141 :param use_sys:
142 boolean indicating whether the sys.modules dictionary should be used or not
143
144 :raise ImportError: if the module or package is not found
145
146 :rtype: module
147 :return: the loaded module
148 """
149 if use_sys:
150 try:
151 return sys.modules['.'.join(parts)]
152 except KeyError:
153 pass
154 modpath = []
155 prevmodule = None
156 for part in parts:
157 modpath.append(part)
158 curname = '.'.join(modpath)
159 module = None
160 if len(modpath) != len(parts):
161
162 module = sys.modules.get(curname)
163 elif use_sys:
164
165 module = sys.modules.get(curname)
166 if module is None:
167 mp_file, mp_filename, mp_desc = find_module(part, path)
168 module = load_module(curname, mp_file, mp_filename, mp_desc)
169 if prevmodule:
170 setattr(prevmodule, part, module)
171 _file = getattr(module, '__file__', '')
172 if not _file and len(modpath) != len(parts):
173 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) )
174 path = [dirname( _file )]
175 prevmodule = module
176 return module
177
178
180 """Load a Python module from it's path.
181
182 :type filepath: str
183 :param filepath: path to the python module or package
184
185 :type path: list or None
186 :param path:
187 optional list of path where the module or package should be
188 searched (use sys.path if nothing or None is given)
189
190 :type use_sys: bool
191 :param use_sys:
192 boolean indicating whether the sys.modules dictionary should be
193 used or not
194
195
196 :raise ImportError: if the module or package is not found
197
198 :rtype: module
199 :return: the loaded module
200 """
201 modpath = modpath_from_file(filepath, extrapath)
202 return load_module_from_modpath(modpath, path, use_sys)
203
204
206 """check there are some __init__.py all along the way"""
207 for part in mod_path:
208 path = join(path, part)
209 if not _has_init(path):
210 return False
211 return True
212
213
215 """given a file path return the corresponding splitted module's name
216 (i.e name of a module or package splitted on '.')
217
218 :type filename: str
219 :param filename: file's path for which we want the module's name
220
221 :type extrapath: dict
222 :param extrapath:
223 optional extra search path, with path as key and package name for the path
224 as value. This is usually useful to handle package splitted in multiple
225 directories using __path__ trick.
226
227
228 :raise ImportError:
229 if the corresponding module's name has not been found
230
231 :rtype: list(str)
232 :return: the corresponding splitted module's name
233 """
234 base = splitext(abspath(filename))[0]
235 if extrapath is not None:
236 for path_ in extrapath:
237 path = abspath(path_)
238 if path and base[:len(path)] == path:
239 submodpath = [pkg for pkg in base[len(path):].split(os.sep)
240 if pkg]
241 if _check_init(path, submodpath[:-1]):
242 return extrapath[path_].split('.') + submodpath
243 for path in sys.path:
244 path = abspath(path)
245 if path and base.startswith(path):
246 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
247 if _check_init(path, modpath[:-1]):
248 return modpath
249 raise ImportError('Unable to find module for %s in %s' % (
250 filename, ', \n'.join(sys.path)))
251
252
253
255 """given a mod path (i.e. splitted module / package name), return the
256 corresponding file, giving priority to source file over precompiled
257 file if it exists
258
259 :type modpath: list or tuple
260 :param modpath:
261 splitted module's name (i.e name of a module or package splitted
262 on '.')
263 (this means explicit relative imports that start with dots have
264 empty strings in this list!)
265
266 :type path: list or None
267 :param path:
268 optional list of path where the module or package should be
269 searched (use sys.path if nothing or None is given)
270
271 :type context_file: str or None
272 :param context_file:
273 context file to consider, necessary if the identifier has been
274 introduced using a relative import unresolvable in the actual
275 context (i.e. modutils)
276
277 :raise ImportError: if there is no such module in the directory
278
279 :rtype: str or None
280 :return:
281 the path to the module's file or None if it's an integrated
282 builtin module such as 'sys'
283 """
284 if context_file is not None:
285 context = dirname(context_file)
286 else:
287 context = context_file
288 if modpath[0] == 'xml':
289
290 try:
291 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
292 except ImportError:
293 return _file_from_modpath(modpath, path, context)
294 elif modpath == ['os', 'path']:
295
296 return os.path.__file__
297 return _file_from_modpath(modpath, path, context)
298
299
300
302 """given a dotted name return the module part of the name :
303
304 >>> get_module_part('logilab.common.modutils.get_module_part')
305 'logilab.common.modutils'
306
307 :type dotted_name: str
308 :param dotted_name: full name of the identifier we are interested in
309
310 :type context_file: str or None
311 :param context_file:
312 context file to consider, necessary if the identifier has been
313 introduced using a relative import unresolvable in the actual
314 context (i.e. modutils)
315
316
317 :raise ImportError: if there is no such module in the directory
318
319 :rtype: str or None
320 :return:
321 the module part of the name or None if we have not been able at
322 all to import the given name
323
324 XXX: deprecated, since it doesn't handle package precedence over module
325 (see #10066)
326 """
327
328 if dotted_name.startswith('os.path'):
329 return 'os.path'
330 parts = dotted_name.split('.')
331 if context_file is not None:
332
333
334 if parts[0] in BUILTIN_MODULES:
335 if len(parts) > 2:
336 raise ImportError(dotted_name)
337 return parts[0]
338
339 path = None
340 starti = 0
341 if parts[0] == '':
342 assert context_file is not None, \
343 'explicit relative import, but no context_file?'
344 path = []
345 starti = 1
346 while parts[starti] == '':
347 starti += 1
348 context_file = dirname(context_file)
349 for i in range(starti, len(parts)):
350 try:
351 file_from_modpath(parts[starti:i+1],
352 path=path, context_file=context_file)
353 except ImportError:
354 if not i >= max(1, len(parts) - 2):
355 raise
356 return '.'.join(parts[:i])
357 return dotted_name
358
359
361 """given a package directory return a list of all available python
362 modules in the package and its subpackages
363
364 :type package: str
365 :param package: the python name for the package
366
367 :type src_directory: str
368 :param src_directory:
369 path of the directory corresponding to the package
370
371 :type blacklist: list or tuple
372 :param blacklist:
373 optional list of files or directory to ignore, default to
374 the value of `logilab.common.STD_BLACKLIST`
375
376 :rtype: list
377 :return:
378 the list of all available python modules in the package and its
379 subpackages
380 """
381 modules = []
382 for directory, dirnames, filenames in os.walk(src_directory):
383 _handle_blacklist(blacklist, dirnames, filenames)
384
385 if not '__init__.py' in filenames:
386 dirnames[:] = ()
387 continue
388 if directory != src_directory:
389 dir_package = directory[len(src_directory):].replace(os.sep, '.')
390 modules.append(package + dir_package)
391 for filename in filenames:
392 if _is_python_file(filename) and filename != '__init__.py':
393 src = join(directory, filename)
394 module = package + src[len(src_directory):-3]
395 modules.append(module.replace(os.sep, '.'))
396 return modules
397
398
399
401 """given a package directory return a list of all available python
402 module's files in the package and its subpackages
403
404 :type src_directory: str
405 :param src_directory:
406 path of the directory corresponding to the package
407
408 :type blacklist: list or tuple
409 :param blacklist:
410 optional list of files or directory to ignore, default to the value of
411 `logilab.common.STD_BLACKLIST`
412
413 :rtype: list
414 :return:
415 the list of all available python module's files in the package and
416 its subpackages
417 """
418 files = []
419 for directory, dirnames, filenames in os.walk(src_directory):
420 _handle_blacklist(blacklist, dirnames, filenames)
421
422 if not '__init__.py' in filenames:
423 dirnames[:] = ()
424 continue
425 for filename in filenames:
426 if _is_python_file(filename):
427 src = join(directory, filename)
428 files.append(src)
429 return files
430
431
433 """given a python module's file name return the matching source file
434 name (the filename will be returned identically if it's a already an
435 absolute path to a python source file...)
436
437 :type filename: str
438 :param filename: python module's file name
439
440
441 :raise NoSourceFile: if no source file exists on the file system
442
443 :rtype: str
444 :return: the absolute path of the source file if it exists
445 """
446 base, orig_ext = splitext(abspath(filename))
447 for ext in PY_SOURCE_EXTS:
448 source_path = '%s.%s' % (base, ext)
449 if exists(source_path):
450 return source_path
451 if include_no_ext and not orig_ext and exists(base):
452 return base
453 raise NoSourceFile(filename)
454
455
457 """remove submodules of `directories` from `sys.modules`"""
458 cleaned = []
459 for modname, module in list(sys.modules.items()):
460 modfile = getattr(module, '__file__', None)
461 if modfile:
462 for directory in directories:
463 if modfile.startswith(directory):
464 cleaned.append(modname)
465 del sys.modules[modname]
466 break
467 return cleaned
468
469
471 """
472 rtype: bool
473 return: True if the filename is a python source file
474 """
475 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
476
477
478
480 """try to guess if a module is a standard python module (by default,
481 see `std_path` parameter's description)
482
483 :type modname: str
484 :param modname: name of the module we are interested in
485
486 :type std_path: list(str) or tuple(str)
487 :param std_path: list of path considered has standard
488
489
490 :rtype: bool
491 :return:
492 true if the module:
493 - is located on the path listed in one of the directory in `std_path`
494 - is a built-in module
495 """
496 modname = modname.split('.')[0]
497 try:
498 filename = file_from_modpath([modname])
499 except ImportError as ex:
500
501
502 return 0
503
504
505 if filename is None:
506 return 1
507 filename = abspath(filename)
508 if filename.startswith(EXT_LIB_DIR):
509 return 0
510 for path in std_path:
511 if filename.startswith(abspath(path)):
512 return 1
513 return False
514
515
516
518 """return true if the given module name is relative to the given
519 file name
520
521 :type modname: str
522 :param modname: name of the module we are interested in
523
524 :type from_file: str
525 :param from_file:
526 path of the module from which modname has been imported
527
528 :rtype: bool
529 :return:
530 true if the module has been imported relatively to `from_file`
531 """
532 if not isdir(from_file):
533 from_file = dirname(from_file)
534 if from_file in sys.path:
535 return False
536 try:
537 find_module(modname.split('.')[0], [from_file])
538 return True
539 except ImportError:
540 return False
541
542
543
544
546 """given a mod path (i.e. splitted module / package name), return the
547 corresponding file
548
549 this function is used internally, see `file_from_modpath`'s
550 documentation for more information
551 """
552 assert len(modpath) > 0
553 if context is not None:
554 try:
555 mtype, mp_filename = _module_file(modpath, [context])
556 except ImportError:
557 mtype, mp_filename = _module_file(modpath, path)
558 else:
559 mtype, mp_filename = _module_file(modpath, path)
560 if mtype == PY_COMPILED:
561 try:
562 return get_source_file(mp_filename)
563 except NoSourceFile:
564 return mp_filename
565 elif mtype == C_BUILTIN:
566
567 return None
568 elif mtype == PKG_DIRECTORY:
569 mp_filename = _has_init(mp_filename)
570 return mp_filename
571
573 for filepath, importer in list(pic.items()):
574 if importer is not None:
575 if importer.find_module(modpath[0]):
576 if not importer.find_module('/'.join(modpath)):
577 raise ImportError('No module named %s in %s/%s' % (
578 '.'.join(modpath[1:]), filepath, modpath))
579 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
580 raise ImportError('No module named %s' % '.'.join(modpath))
581
582 try:
583 import pkg_resources
584 except ImportError:
585 pkg_resources = None
586
588 """get a module type / file path
589
590 :type modpath: list or tuple
591 :param modpath:
592 splitted module's name (i.e name of a module or package splitted
593 on '.'), with leading empty strings for explicit relative import
594
595 :type path: list or None
596 :param path:
597 optional list of path where the module or package should be
598 searched (use sys.path if nothing or None is given)
599
600
601 :rtype: tuple(int, str)
602 :return: the module type flag and the file path for a module
603 """
604
605 try:
606 pic = sys.path_importer_cache
607 _path = (path is None and sys.path or path)
608 for __path in _path:
609 if not __path in pic:
610 try:
611 pic[__path] = zipimport.zipimporter(__path)
612 except zipimport.ZipImportError:
613 pic[__path] = None
614 checkeggs = True
615 except AttributeError:
616 checkeggs = False
617
618 if pkg_resources is not None and modpath[0] in pkg_resources._namespace_packages and len(modpath) > 1:
619
620
621 module = sys.modules[modpath.pop(0)]
622 path = module.__path__
623 imported = []
624 while modpath:
625 modname = modpath[0]
626
627
628
629
630
631
632
633
634
635 try:
636 _, mp_filename, mp_desc = find_module(modname, path)
637 except ImportError:
638 if checkeggs:
639 return _search_zip(modpath, pic)[:2]
640 raise
641 else:
642 if checkeggs and mp_filename:
643 fullabspath = [abspath(x) for x in _path]
644 try:
645 pathindex = fullabspath.index(dirname(abspath(mp_filename)))
646 emtype, emp_filename, zippath = _search_zip(modpath, pic)
647 if pathindex > _path.index(zippath):
648
649 return emtype, emp_filename
650 except ValueError:
651
652 pass
653 except ImportError:
654 pass
655 checkeggs = False
656 imported.append(modpath.pop(0))
657 mtype = mp_desc[2]
658 if modpath:
659 if mtype != PKG_DIRECTORY:
660 raise ImportError('No module %s in %s' % ('.'.join(modpath),
661 '.'.join(imported)))
662
663
664 try:
665 with open(join(mp_filename, '__init__.py')) as stream:
666 data = stream.read(4096)
667 except IOError:
668 path = [mp_filename]
669 else:
670 if 'pkgutil' in data and 'extend_path' in data:
671
672
673 path = [join(p, *imported) for p in sys.path
674 if isdir(join(p, *imported))]
675 else:
676 path = [mp_filename]
677 return mtype, mp_filename
678
680 """return true if the given filename should be considered as a python file
681
682 .pyc and .pyo are ignored
683 """
684 for ext in ('.py', '.so', '.pyd', '.pyw'):
685 if filename.endswith(ext):
686 return True
687 return False
688
689
691 """if the given directory has a valid __init__ file, return its path,
692 else return None
693 """
694 mod_or_pack = join(directory, '__init__')
695 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
696 if exists(mod_or_pack + '.' + ext):
697 return mod_or_pack + '.' + ext
698 return None
699