tensorflow/keras/numpy breaks pydeps

Question

tensorflow/keras/numpy breaks pydeps

Closed this issue 2 years ago · 3 comments

Have tried it in both native and virtualenvs. I have also tried downloading tensorflow sourcecode and running pydeps on source code but the same uncaught exception appears.

$ pydeps --version
pydeps v1.10.14

$ python --version
Python 3.8.10

$ pydeps --collapse-target ~/.virtualenvs/ast-catch/lib/python3.8/site-packages/tensorflow --include-missing

Traceback (most recent call last):
File "/home/ajinkya/.virtualenvs/ast-catch/bin/pydeps", line 8, in
sys.exit(pydeps())
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/pydeps.py", line 150, in pydeps
return _pydeps(inp, **_args)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/pydeps.py", line 32, in _pydeps
dep_graph = py2depgraph.py2dep(trgt, **kw)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 214, in py2dep
mf.run_script(dummy.fname)
File "/usr/lib/python3.8/modulefinder.py", line 163, in run_script
self.load_module('main', fp, pathname, stuff)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 145, in load_module
module = mf27.ModuleFinder.load_module(
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 75, in load_module
self.scan_code(co, m)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 103, in scan_code
self._safe_import_hook(name, m, fromlist, level=level)
File "/usr/lib/python3.8/modulefinder.py", line 377, in _safe_import_hook
self.import_hook(name, caller, level=level)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 116, in import_hook
return mf27.ModuleFinder.import_hook(self, name, caller, fromlist, level)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 20, in import_hook
q, tail = self.find_head_package(parent, name)
File "/usr/lib/python3.8/modulefinder.py", line 231, in find_head_package
q = self.import_module(head, qname, parent)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 138, in import_module
module = mf27.ModuleFinder.import_module(self, partnam, fqname, parent)
File "/usr/lib/python3.8/modulefinder.py", line 325, in import_module
m = self.load_module(fqname, fp, pathname, stuff)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 145, in load_module
module = mf27.ModuleFinder.load_module(
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 41, in load_module
module = self.load_package(fqname, pathname)
File "/usr/lib/python3.8/modulefinder.py", line 481, in load_package
self.load_module(fqname, fp, buf, stuff)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 145, in load_module
module = mf27.ModuleFinder.load_module(
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 75, in load_module
self.scan_code(co, m)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 103, in scan_code
self._safe_import_hook(name, m, fromlist, level=level)
File "/usr/lib/python3.8/modulefinder.py", line 377, in _safe_import_hook
self.import_hook(name, caller, level=level)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 116, in import_hook
return mf27.ModuleFinder.import_hook(self, name, caller, fromlist, level)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 23, in import_hook
m = self.load_tail(q, tail)
File "/usr/lib/python3.8/modulefinder.py", line 253, in load_tail
m = self.import_module(head, mname, m)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 138, in import_module
module = mf27.ModuleFinder.import_module(self, partnam, fqname, parent)
File "/usr/lib/python3.8/modulefinder.py", line 325, in import_module
m = self.load_module(fqname, fp, pathname, stuff)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 145, in load_module
module = mf27.ModuleFinder.load_module(
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 41, in load_module
module = self.load_package(fqname, pathname)
File "/usr/lib/python3.8/modulefinder.py", line 481, in load_package
self.load_module(fqname, fp, buf, stuff)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 145, in load_module
module = mf27.ModuleFinder.load_module(
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 75, in load_module
self.scan_code(co, m)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 103, in scan_code
self._safe_import_hook(name, m, fromlist, level=level)
File "/usr/lib/python3.8/modulefinder.py", line 392, in _safe_import_hook
self.import_hook(name, caller, [sub], level=level)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 116, in import_hook
return mf27.ModuleFinder.import_hook(self, name, caller, fromlist, level)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 27, in import_hook
self.ensure_fromlist(m, fromlist)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 171, in ensure_fromlist
submod = self.import_module(sub, subname, module)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 138, in import_module
module = mf27.ModuleFinder.import_module(self, partnam, fqname, parent)
File "/usr/lib/python3.8/modulefinder.py", line 325, in import_module
m = self.load_module(fqname, fp, pathname, stuff)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 145, in load_module
module = mf27.ModuleFinder.load_module(
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 75, in load_module
self.scan_code(co, m)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 103, in scan_code
self._safe_import_hook(name, m, fromlist, level=level)
File "/usr/lib/python3.8/modulefinder.py", line 392, in _safe_import_hook
self.import_hook(name, caller, [sub], level=level)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 116, in import_hook
return mf27.ModuleFinder.import_hook(self, name, caller, fromlist, level)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 27, in import_hook
self.ensure_fromlist(m, fromlist)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 171, in ensure_fromlist
submod = self.import_module(sub, subname, module)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 138, in import_module
module = mf27.ModuleFinder.import_module(self, partnam, fqname, parent)
File "/usr/lib/python3.8/modulefinder.py", line 325, in import_module
m = self.load_module(fqname, fp, pathname, stuff)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 145, in load_module
module = mf27.ModuleFinder.load_module(
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 75, in load_module
self.scan_code(co, m)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 103, in scan_code
self._safe_import_hook(name, m, fromlist, level=level)
File "/usr/lib/python3.8/modulefinder.py", line 377, in _safe_import_hook
self.import_hook(name, caller, level=level)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 116, in import_hook
return mf27.ModuleFinder.import_hook(self, name, caller, fromlist, level)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/mf27.py", line 20, in import_hook
q, tail = self.find_head_package(parent, name)
File "/usr/lib/python3.8/modulefinder.py", line 231, in find_head_package
q = self.import_module(head, qname, parent)
File "/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/pydeps/py2depgraph.py", line 138, in import_module
module = mf27.ModuleFinder.import_module(self, partnam, fqname, parent)
File "/usr/lib/python3.8/modulefinder.py", line 318, in import_module
fp, pathname, stuff = self.find_module(partname,
File "/usr/lib/python3.8/modulefinder.py", line 510, in find_module
return _find_module(name, path)
File "/usr/lib/python3.8/modulefinder.py", line 79, in _find_module
if spec.loader.is_package(name):
AttributeError: 'NoneType' object has no attribute 'is_package'

Here is the pdb output of the spec which has loader attribute set to None

ModuleSpec(name='google', loader=None, submodule_search_locations=_NamespacePath(['/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/google', '/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/google', '/home/ajinkya/.virtualenvs/ast-catch/lib/python3.8/site-packages/google']))

Answer 1 · 2022-04-21T11:55:21.000Z

Hmm... seems to be a problem in Python's modulefinder:

(pydeps141) bp@bp-222:~/envs/pydeps141/lib/python3.8/site-packages$ cat foo.py
from tensorflow import *
(pydeps141) bp@bp-222:~/envs/pydeps141/lib/python3.8/site-packages$ python -m modulefinder foo.py
...
  File "/usr/lib/python3.8/modulefinder.py", line 231, in find_head_package
    q = self.import_module(head, qname, parent)
  File "/usr/lib/python3.8/modulefinder.py", line 318, in import_module
    fp, pathname, stuff = self.find_module(partname,
  File "/usr/lib/python3.8/modulefinder.py", line 510, in find_module
    return _find_module(name, path)
  File "/usr/lib/python3.8/modulefinder.py", line 79, in _find_module
    if spec.loader.is_package(name):
AttributeError: 'NoneType' object has no attribute 'is_package'

Answer 2 · 2022-04-21T12:02:03.000Z

It was reported as bug 84530 (python/cpython#84530). It seems to have a PR, but it has yet to be merged.

Answer 3 · 2022-04-21T16:23:47.000Z

This should be fixed in the latest version (v1.10.17) available on PyPI.