pysource.py

#!/usr/bin/env python

"""\
List python source files.

There are three functions to check whether a file is a Python source, listed
here with increasing complexity:

- has_python_ext() checks whether a file name ends in '.py[w]'.
- look_like_python() checks whether the file is not binary and either has
 the '.py[w]' extension or the first line contains the word 'python'.
- can_be_compiled() checks whether the file can be compiled by compile().

The file also must be of appropriate size - not bigger than a megabyte.

walk_python_files() recursively lists all Python files under the given directories.
"""
__author__="Oleg Broytmann, Georg Brandl"

__all__= ["has_python_ext", "looks_like_python", "can_be_compiled", "walk_python_files"]


importos, re

binary_re=re.compile('[\x00-\x08\x0E-\x1F\x7F]')

debug=False

defprint_debug(msg):
ifdebug: printmsg


def_open(fullpath):
try:
size=os.stat(fullpath).st_size
exceptOSError, err: # Permission denied - ignore the file
print_debug("%s: permission denied: %s"% (fullpath, err))
returnNone

ifsize>1024*1024: # too big
print_debug("%s: the file is too big: %d bytes"% (fullpath, size))
returnNone

try:
returnopen(fullpath, 'rU')
exceptIOError, err: # Access denied, or a special file - ignore it
print_debug("%s: access denied: %s"% (fullpath, err))
returnNone

defhas_python_ext(fullpath):
returnfullpath.endswith(".py") orfullpath.endswith(".pyw")

deflooks_like_python(fullpath):
infile=_open(fullpath)
ifinfileisNone:
returnFalse

line=infile.readline()
infile.close()

ifbinary_re.search(line):
# file appears to be binary
print_debug("%s: appears to be binary"%fullpath)
returnFalse

iffullpath.endswith(".py") orfullpath.endswith(".pyw"):
returnTrue
elif"python"inline:
# disguised Python script (e.g. CGI)
returnTrue

returnFalse

defcan_be_compiled(fullpath):
infile=_open(fullpath)
ifinfileisNone:
returnFalse

code=infile.read()
infile.close()

try:
compile(code, fullpath, "exec")
exceptException, err:
print_debug("%s: cannot compile: %s"% (fullpath, err))
returnFalse

returnTrue


defwalk_python_files(paths, is_python=looks_like_python, exclude_dirs=None):
"""\
 Recursively yield all Python source files below the given paths.

 paths: a list of files and/or directories to be checked.
 is_python: a function that takes a file name and checks whether it is a
 Python source file
 exclude_dirs: a list of directory base names that should be excluded in
 the search
 """
ifexclude_dirsisNone:
exclude_dirs=[]

forpathinpaths:
print_debug("testing: %s"%path)
ifos.path.isfile(path):
ifis_python(path):
yieldpath
elifos.path.isdir(path):
print_debug(" it is a directory")
fordirpath, dirnames, filenamesinos.walk(path):
forexcludeinexclude_dirs:
ifexcludeindirnames:
dirnames.remove(exclude)
forfilenameinfilenames:
fullpath=os.path.join(dirpath, filename)
print_debug("testing: %s"%fullpath)
ifis_python(fullpath):
yieldfullpath
else:
print_debug(" unknown type")


if__name__=="__main__":
# Two simple examples/tests
forfullpathinwalk_python_files(['.']):
printfullpath
print"----------"
forfullpathinwalk_python_files(['.'], is_python=can_be_compiled):
printfullpath