"""Parse a Python module and describe its classes and functions.

Parse enough of a Python file to recognize imports and class and

function definitions, and to find out the superclasses of a class.

The interface consists of a single function:

readmodule_ex(module, path=None)

where module is the name of a Python module, and path is an optional

list of directories where the module is to be searched. If present,

path is prepended to the system search path sys.path. The return value

is a dictionary. The keys of the dictionary are the names of the

classes and functions defined in the module (including classes that are

defined via the from XXX import YYY construct). The values are

instances of classes Class and Function. One special key/value pair is

present for packages: the key '__path__' has a list as its value which

contains the package search path.

Classes and Functions have a common superclass: _Object. Every instance

has the following attributes:

module -- name of the module;

name -- name of the object;

file -- file in which the object is defined;

lineno -- line in the file where the object's definition starts;

parent -- parent of this object, if any;

children -- nested objects contained in this object.

The 'children' attribute is a dictionary mapping names to objects.

Instances of Function describe functions with the attributes from _Object.

Instances of Class describe classes with the attributes from _Object,

plus the following:

super -- list of super classes (Class instances if possible);

methods -- mapping of method names to beginning line numbers.

If the name of a super class is not recognized, the corresponding

entry in the list of super classes is not a class instance but a

string giving the name of the super class. Since import statements

are recognized and imported modules are scanned as well, this

shouldn't happen often.

"""

import io

import sys

import importlib.util

import tokenize

from token import NAME, DEDENT, OP

__all__ = ["readmodule", "readmodule_ex", "Class", "Function"]

_modules = {} # Initialize cache of modules we've seen.

class _Object:

"Information about Python class or function."

def __init__(self, module, name, file, lineno, parent):

self.module = module

self.name = name

self.file = file

self.lineno = lineno

self.parent = parent

self.children = {}

def _addchild(self, name, obj):

self.children[name] = obj

class Function(_Object):

"Information about a Python function, including methods."

def __init__(self, module, name, file, lineno, parent=None):

_Object.__init__(self, module, name, file, lineno, parent)

class Class(_Object):

"Information about a Python class."

def __init__(self, module, name, super, file, lineno, parent=None):

_Object.__init__(self, module, name, file, lineno, parent)

self.super = [] if super is None else super

self.methods = {}

def _addmethod(self, name, lineno):

self.methods[name] = lineno

def _nest_function(ob, func_name, lineno):

"Return a Function after nesting within ob."

newfunc = Function(ob.module, func_name, ob.file, lineno, ob)

ob._addchild(func_name, newfunc)

if isinstance(ob, Class):

ob._addmethod(func_name, lineno)

return newfunc

def _nest_class(ob, class_name, lineno, super=None):

"Return a Class after nesting within ob."

newclass = Class(ob.module, class_name, super, ob.file, lineno, ob)

ob._addchild(class_name, newclass)

return newclass

def readmodule(module, path=None):

"""Return Class objects for the top-level classes in module.

This is the original interface, before Functions were added.

"""

res = {}

for key, value in _readmodule(module, path or []).items():

if isinstance(value, Class):

res[key] = value

return res

def readmodule_ex(module, path=None):

"""Return a dictionary with all functions and classes in module.

Search for module in PATH + sys.path.

If possible, include imported superclasses.

Do this by reading source, without importing (and executing) it.

"""

return _readmodule(module, path or [])

def _readmodule(module, path, inpackage=None):

"""Do the hard work for readmodule[_ex].

If inpackage is given, it must be the dotted name of the package in

which we are searching for a submodule, and then PATH must be the

package search path; otherwise, we are searching for a top-level

module, and path is combined with sys.path.

"""

# Compute the full module name (prepending inpackage if set).

if inpackage is not None:

fullmodule = "%s.%s" % (inpackage, module)

else:

fullmodule = module

# Check in the cache.

if fullmodule in _modules:

return _modules[fullmodule]

# Initialize the dict for this module's contents.

tree = {}

# Check if it is a built-in module; we don't do much for these.

if module in sys.builtin_module_names and inpackage is None:

_modules[module] = tree

return tree

# Check for a dotted module name.

i = module.rfind('.')

if i >= 0:

package = module[:i]

submodule = module[i+1:]

parent = _readmodule(package, path, inpackage)

if inpackage is not None:

package = "%s.%s" % (inpackage, package)

if not '__path__' in parent:

raise ImportError('No package named {}'.format(package))

return _readmodule(submodule, parent['__path__'], package)

# Search the path for the module.

f = None

if inpackage is not None:

search_path = path

else:

search_path = path + sys.path

spec = importlib.util._find_spec_from_path(fullmodule, search_path)

if spec is None:

raise ModuleNotFoundError(f"no module named {fullmodule!r}", name=fullmodule)

_modules[fullmodule] = tree

# Is module a package?

if spec.submodule_search_locations is not None:

tree['__path__'] = spec.submodule_search_locations

try:

source = spec.loader.get_source(fullmodule)

except (AttributeError, ImportError):

# If module is not Python source, we cannot do anything.

return tree

else:

if source is None:

return tree

fname = spec.loader.get_filename(fullmodule)

return _create_tree(fullmodule, path, fname, source, tree, inpackage)

def _create_tree(fullmodule, path, fname, source, tree, inpackage):

"""Return the tree for a particular module.

fullmodule (full module name), inpackage+module, becomes o.module.

path is passed to recursive calls of _readmodule.

fname becomes o.file.

source is tokenized. Imports cause recursive calls to _readmodule.

tree is {} or {'__path__': <submodule search locations>}.

inpackage, None or string, is passed to recursive calls of _readmodule.

The effect of recursive calls is mutation of global _modules.

"""

f = io.StringIO(source)

stack = [] # Initialize stack of (class, indent) pairs.

g = tokenize.generate_tokens(f.readline)

try:

for tokentype, token, start, _end, _line in g:

if tokentype == DEDENT:

lineno, thisindent = start

# Close previous nested classes and defs.

while stack and stack[-1][1] >= thisindent:

del stack[-1]

elif token == 'def':

lineno, thisindent = start

# Close previous nested classes and defs.

while stack and stack[-1][1] >= thisindent:

del stack[-1]

tokentype, func_name, start = next(g)[0:3]

if tokentype != NAME:

continue # Skip def with syntax error.

cur_func = None

if stack:

cur_obj = stack[-1][0]

cur_func = _nest_function(cur_obj, func_name, lineno)

else:

# It is just a function.

cur_func = Function(fullmodule, func_name, fname, lineno)

tree[func_name] = cur_func

stack.append((cur_func, thisindent))

elif token == 'class':

lineno, thisindent = start

# Close previous nested classes and defs.

while stack and stack[-1][1] >= thisindent:

del stack[-1]

tokentype, class_name, start = next(g)[0:3]

if tokentype != NAME:

continue # Skip class with syntax error.

# Parse what follows the class name.

tokentype, token, start = next(g)[0:3]

inherit = None

if token == '(':

names = [] # Initialize list of superclasses.

level = 1

super = [] # Tokens making up current superclass.

while True:

tokentype, token, start = next(g)[0:3]

if token in (')', ',') and level == 1:

n = "".join(super)

if n in tree:

# We know this super class.

n = tree[n]

else:

c = n.split('.')

if len(c) > 1:

# Super class form is module.class:

# look in module for class.

m = c[-2]

c = c[-1]

if m in _modules:

d = _modules[m]

if c in d:

n = d[c]

names.append(n)

super = []

if token == '(':

level += 1

elif token == ')':

level -= 1

if level == 0:

break

elif token == ',' and level == 1:

pass

# Only use NAME and OP (== dot) tokens for type name.

elif tokentype in (NAME, OP) and level == 1:

super.append(token)

# Expressions in the base list are not supported.

inherit = names

if stack:

cur_obj = stack[-1][0]

cur_class = _nest_class(

cur_obj, class_name, lineno, inherit)

else:

cur_class = Class(fullmodule, class_name, inherit,

fname, lineno)

tree[class_name] = cur_class

stack.append((cur_class, thisindent))

elif token == 'import' and start[1] == 0:

modules = _getnamelist(g)

for mod, _mod2 in modules:

try:

# Recursively read the imported module.

if inpackage is None:

_readmodule(mod, path)

else:

try:

_readmodule(mod, path, inpackage)

except ImportError:

_readmodule(mod, [])

except:

# If we can't find or parse the imported module,

# too bad -- don't die here.

pass

elif token == 'from' and start[1] == 0:

mod, token = _getname(g)

if not mod or token != "import":

continue

names = _getnamelist(g)

try:

# Recursively read the imported module.

d = _readmodule(mod, path, inpackage)

except:

# If we can't find or parse the imported module,

# too bad -- don't die here.

continue

# Add any classes that were defined in the imported module

# to our name space if they were mentioned in the list.

for n, n2 in names:

if n in d:

tree[n2 or n] = d[n]

elif n == '*':

# Don't add names that start with _.

for n in d:

if n[0] != '_':

tree[n] = d[n]

except StopIteration:

pass

f.close()

return tree

def _getnamelist(g):

"""Return list of (dotted-name, as-name or None) tuples for token source g.

An as-name is the name that follows 'as' in an as clause.

"""

names = []

while True:

name, token = _getname(g)

if not name:

break

if token == 'as':

name2, token = _getname(g)

else:

name2 = None

names.append((name, name2))

while token != "," and "\n" not in token:

token = next(g)[1]

if token != ",":

break

return names

def _getname(g):

"Return (dotted-name or None, next-token) tuple for token source g."

parts = []

tokentype, token = next(g)[0:2]

if tokentype != NAME and token != '*':

return (None, token)

parts.append(token)

while True:

tokentype, token = next(g)[0:2]

if token != '.':

break

tokentype, token = next(g)[0:2]

if tokentype != NAME:

break

parts.append(token)

return (".".join(parts), token)

def _main():

"Print module output (default this file) for quick visual check."

import os

try:

mod = sys.argv[1]

except:

mod = __file__

if os.path.exists(mod):

path = [os.path.dirname(mod)]

mod = os.path.basename(mod)

if mod.lower().endswith(".py"):

mod = mod[:-3]

else:

path = []

tree = readmodule_ex(mod, path)

lineno_key = lambda a: getattr(a, 'lineno', 0)

objs = sorted(tree.values(), key=lineno_key, reverse=True)

indent_level = 2

while objs:

obj = objs.pop()

if isinstance(obj, list):

# Value is a __path__ key.

continue

if not hasattr(obj, 'indent'):

obj.indent = 0

if isinstance(obj, _Object):

new_objs = sorted(obj.children.values(),

key=lineno_key, reverse=True)

for ob in new_objs:

ob.indent = obj.indent + indent_level

objs.extend(new_objs)

if isinstance(obj, Class):

print("{}class {} {} {}"

.format(' ' * obj.indent, obj.name, obj.super, obj.lineno))

elif isinstance(obj, Function):

print("{}def {} {}".format(' ' * obj.indent, obj.name, obj.lineno))

if __name__ == "__main__":

_main()