#!/usr/bin/env python
#
# $Id: __init__.py,v 1.6 2006/12/05 13:10:46 doughellmann Exp $
#
# Copyright 2001 Doug Hellmann.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Doug
# Hellmann not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
"""Extract information from a Python code parse tree.
This module is based on the Demos/parser/example.py module
distributed with the Python source distribution.
File-specific Parser Configuration Values
Parameters to the HappyDoc Parser can also be embedded within the
first comment block of the module. The parameter values
recognized and their meanings are listed below.
To provide file-specific parser configuration settings, any Python
code can be embedded in the comments of the file. For example::
#!/usr/bin/env python
#
# HappyDoc:# These variables should be discovered.
# HappyDoc:TestInt=1
# HappyDoc:TestString="String"
# HappyDoc:TestStringModule=string.strip(' this has spaces in front and back ')
# HappyDoc:url=urlencode({'a':'A', 'b':'B'})
# HappyDoc:docStringFormat='StructuredText'
All lines beginning with the pattern "'# HappyDoc:'" will be
concatenated (separated by newlines) and 'execed'. The local
namespace resulting from the execution of the code will be
examined for variables of interest to the parser. The incoming
global namespace for the configuration code will have a few
pre-populated names for convenience.
Pre-defined Globals
|------------------------------------------------------------------|
| Name | Description |
|==================================================================|
| string | The 'string' module. |
|------------------------------------------------------------------|
| urlquote | Same as 'urllib.quote' function. |
|------------------------------------------------------------------|
| urlencode | Same as 'urllib.urlencode' function. |
|------------------------------------------------------------------|
Recognized Parser Configuration Variables
|------------------------------------------------------------------|
| Parameter | Description |
|==================================================================|
| docStringFormat | The name of the format for the '__doc__' |
| | strings in the module. This value is used |
| | to determine the docstring converter which |
| | will know how to translate the docstrings in |
| | the module. |
|------------------------------------------------------------------|
"""
__rcs_info__ = {
#
# Creation Information
#
'module_name' : '$RCSfile: __init__.py,v $',
'rcs_id' : '$Id: __init__.py,v 1.6 2006/12/05 13:10:46 doughellmann Exp $',
'creator' : 'Doug Hellmann',
'project' : 'HappyDoc',
'created' : 'Sun, 11-Nov-2001 10:44:25 EST',
#
# Current Information
#
'author' : '$Author: doughellmann $',
'version' : '$Revision: 1.6 $',
'date' : '$Date: 2006/12/05 13:10:46 $',
}
try:
__version__ = __rcs_info__['version'].split(' ')[1]
except:
__version__ = '0.0'
#
# Import system modules
#
import md5
import os
import parser
try:
import cPickle as Pickle
except:
import Pickle
import types
import unittest
#
# Import Local modules
#
import happydoclib
import parsecomments
from happydoclib.parseinfo.moduleinfo import ModuleInfo
#
# Module
#
TRACE_LEVEL=2
_parser_options = {
'include_comments':1,
'docStringFormat':'StructuredText',
'useCache':1,
'cacheFilePrefix':'.happydoc.',
}
def setOption(**kwargs):
"""Set options to control the parser behavior.
Options
include_comments -- Boolean (default true) controlling whether
text should be taken from comments in
addition to docstrings.
"""
global _parser_options
#print 'Before %s' % str(_parser_options)
#print 'Updating with %s' % str(kwargs)
_parser_options.update(kwargs)
#print 'After %s' % str(_parser_options)
return
def getOption(name):
"""Get options which control the parser behavior.
Options
name -- The name of the option whose value should be retrieved.
"""
global _parser_options
return _parser_options[name]
def getDocs(parent, fileName):
global _parser_options
if _parser_options['useCache']:
return getDocsFromCache(parent, fileName)
else:
return getDocsFromFile(parent, fileName)
def getDocsFromCache(parent, fileName):
happydoclib.TRACE.into('parseinfo', 'getDocsFromCache',
parent=parent,
fileName=fileName,
outputLevel=TRACE_LEVEL,
)
#
# Check for cached parse results
#
# cache_file_name = '%s.happydoc' % name
dir_name = os.path.dirname(fileName)
base_name = os.path.basename(fileName)
cache_file_prefix = _parser_options['cacheFilePrefix']
cache_file_name = os.path.join(dir_name, '%s%s' % (cache_file_prefix,
base_name)
)
cached_md5 = None
cached_mod_info = None
if os.path.exists(cache_file_name):
try:
cache_file = open(cache_file_name, 'rb')
except IOError:
pass
else:
try:
cached_md5, cached_mod_info = Pickle.load(cache_file)
except EOFError:
pass
cache_file.close()
#
# Compute our md5 digest to determine
# whether or not we should
# use the cached results.
#
m = md5.new()
input_file = open(fileName, 'r')
m.update(input_file.read())
input_file.close()
current_md5 = m.hexdigest()
if current_md5 == cached_md5:
mod_info = cached_mod_info
else:
mod_info = getDocsFromFile(parent, fileName)
#
# Write back to the cache
#
cache_file = open(cache_file_name, 'wb')
Pickle.dump( (current_md5, mod_info),
cache_file )
cache_file.close()
happydoclib.TRACE.outof(mod_info,
outputLevel=TRACE_LEVEL,
)
return mod_info
def getDocsFromFile(parent, fileName):
"""Retrieve information from the parse tree of a source file.
Parameters
fileName --
Name of the file to read Python source code from.
"""
happydoclib.TRACE.into('parseinfo', 'getDocsFromFile',
parent=parent,
fileName=fileName,
outputLevel=TRACE_LEVEL,
)
f = open(fileName)
#
# Read file and add an extra newline to fix problem
# reported with files containing only a single docstring
# line.
#
source = f.read()
source = '\n'.join(source.split('\r\n')) + '\n'
f.close()
base_filename = os.path.basename(os.path.splitext(fileName)[0])
try:
ast = parser.suite(source)
except parser.ParserError, msg:
#
# Catch parse exception and retry with the
# compile function which produces better user
# error messages.
#
code = compile(source, fileName, 'exec')
#
# In case the compile module can do something,
# just re-raise the exception we got from the
# parser.
#
raise
except TypeError, msg:
raise TypeError(msg, source)
tup = parser.ast2tuple(ast)
include_comments = getOption('include_comments')
if include_comments:
comment_info = parsecomments.extractComments(source)
else:
comment_info = {}
happydoclib.TRACE.write('Creating ModuleInfo',
outputLevel=TRACE_LEVEL,
)
config_values = {}
global _parser_options
config_values.update(_parser_options)
mod_info = ModuleInfo(parent=parent,
tree=tup,
name=base_filename,
fileName=fileName,
commentInfo=comment_info,
defaultConfigValues=config_values)
happydoclib.TRACE.outof(mod_info,
outputLevel=TRACE_LEVEL,
)
return mod_info
################################################################################
class ParserUnitTest(unittest.TestCase):
default_filename = './TestCases/test.py'
filename_map = {
'testExtractVariablesFromModule':'./TestCases/parseinfo/test_variable_extraction.py',
'testExtractVariablesFromModuleWithException':'./TestCases/parseinfo/test_variable_extraction_with_exception.py',
'testVariousImportStatements':'./TestCases/parseinfo/test_import_statements.py',
'testIgnoreComments':'TestCases/parseinfo/test_ignore_comments.py',
'testDecoratedComments':'TestCases/parseinfo/test_decorated_comments.py',
'testDOSFiles':'TestCases/parseinfo/test_bug434946.py',
'testEmptyModule':'TestCases/parseinfo/emptytest.py',
'testFunctionParameters':'TestCases/parseinfo/test_function_params.py',
'testNestedClasses':'TestCases/parseinfo/test_nested_structures.py',
'testNestedFunctions':'TestCases/parseinfo/test_nested_structures.py',
}
def setUp(self):
name = self._TestCase__testMethodName
filename = self.filename_map.get(name, self.default_filename)
self.filename = filename
setOption(include_comments=1)
setOption(useCache=0)
self.parsed_module = getDocs(None, filename)
return
def tearDown(self):
setOption(useCache=1)
unittest.TestCase.tearDown(self)
return
def _docStringsAreEqual(self, ds1, ds2):
if not ds1 and not ds2:
return 1
if cmp(ds1, ds2):
return 0
else:
return 1
def _testComparison(self, full_name, name, importedInfo, parsedInfo, allowedFailures=[]):
if name[:2] == '__':
# Ignore private names
return
#
# Get the thing from the imported module
#
obj = getattr(importedInfo, name)
if type(obj) == types.IntType:
return
elif type(obj) == types.ClassType:
if name not in parsedInfo._class_info.keys():
if name in allowedFailures:
return
assert name in parsedInfo._class_info.keys(), \
'Did not find class docs for %s' % full_name
try:
aClass = parsedInfo[name]
except KeyError:
self.fail('Did not find class info for %s' % full_name)
#
# Verify the class info docstring matches the parser results
#
assert self._docStringsAreEqual(aClass._docstring, obj.__doc__), \
'Docs for class %s do not match (imported "%s", parsed "%s")' % \
(full_name, obj.__doc__, aClass._docstring)
#
# Recurse to look at methods
#
# Importing results in more names than
# parsing, so this does not work.
#
#for attr_name in dir(obj):
# self._testComparison('%s.%s' % (full_name, attr_name),
# attr_name,
# obj,
# aClass,
# allowedFailures
# )
elif type(obj) == types.MethodType:
assert name in parsedInfo._function_info.keys(), \
'Did not find method info for %s' % full_name
aMethod = parsedInfo[name]
assert self._docStringsAreEqual(aMethod._docstring, obj.__doc__), \
'Docs for method %s do not match (imported "%s", parsed "%s")' % \
(full_name, obj.__doc__, aMethod._docstring)
elif type(obj) == types.FunctionType:
if name not in parsedInfo._function_info.keys():
if name in allowedFailures:
return
assert name in parsedInfo._function_info.keys(), \
'Did not find func info for %s' % full_name
aFunc = parsedInfo[name]
assert self._docStringsAreEqual(aFunc._docstring, obj.__doc__), \
'Docs for func %s do not match (imported "%s", parsed "%s")' % \
(full_name, obj.__doc__, aFunc._docstring)
else:
assert obj, 'Got %s for %s' % (obj, full_name)
return
def testBasicParser(self):
package_name = 'TestCases.test'
temp_locals = {}
#
# This test expects to be run in the root of the HappyDoc
# source tree (parent dir of happydoclib).
#
import sys, os
sys.path.append(os.getcwd())
imported_module = __import__(package_name, globals(), temp_locals)
assert imported_module, 'Could not import %s' % filename
test_module = imported_module.test
for name in dir(test_module):
full_name = '%s.%s' % (package_name, name)
self._testComparison(full_name, name, test_module, self.parsed_module,
['OuterClass', 'OuterFunction', 'TestApp',
'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', 'main',
'appInit'
])
return
def testExtractVariablesFromModule(self):
expected_values = {
'TestInt':1,
'TestString':"String",
'TestStringModule':"this has spaces in front and back",
'url': 'a=A&b=B',
}
expected_values.update(_parser_options)
module_values = self.parsed_module.getConfigurationValues()
#for name, value in module_values.items():
# self.failUnlessEqual(
# value, expected_values[name],
# 'Got %s for %s instead of %s' % (value, name, expected_values[name])
# )
# if self.verboseLevel.get() > 1:
# print 'Module variables for %s' % self.filename
# import pprint
# pprint.pprint(module_values)
self.failUnlessEqual(module_values, expected_values)
return
def testExtractVariablesFromModuleWithException(self):
expected_values = {
}
expected_values.update(_parser_options)
module_values = self.parsed_module.getConfigurationValues()
self.failUnlessEqual(module_values, expected_values)
#assert not module_values, 'Did not find expected exception'
return
def testVariousImportStatements(self):
expected_import_data = [
('CommandLineApp', None),
('CommandLineApp', ['TestApp', 'SubClassTestApp']),
('Module1', None),
('Module2', ['Symbol1']),
('Package1.SubModule1', None),
('Package2.SubModule2', ['Symbol2']),
('Package3.SubModule3', ['Symbol3']),
('a.b', ['c', 'd']),
('cgi', None),
('one.two', None),
('os', None),
('prettyast', ['astListFixNames']),
('string', None),
('string', ['strip']),
('sys', ['path']),
('token', ['*']),
('types', None),
('webbrowser', None),
]
import_data = self.parsed_module.getImportData()
import_data.sort()
assert import_data, 'No imports were found.'
for expected, actual in map(None, expected_import_data, import_data):
if not expected or not actual:
break
assert expected == actual, 'Import values do not match %s vs. %s' % \
(expected, actual)
assert import_data == expected_import_data, 'Did not find expected values, got %s instead' % str(import_data)
return
def testIgnoreCommentsSetting(self):
setOption(include_comments=0)
include_comments = getOption('include_comments')
if include_comments:
self.fail('include_comments option not turned off (%s)' % include_comments)
return
def testIncludeCommentsSetting(self):
setOption(include_comments=1)
include_comments = getOption('include_comments')
if not include_comments:
self.fail('include_comments option not turned off')
return
def testIgnoreComments(self):
setOption(include_comments=0)
parsed_module = getDocs(None, self.filename)
assert not parsed_module._comments, \
'Did not ignore module comments %s' % self.filename
assert parsed_module._docstring, \
'Did not find docstring for module %s' % self.filename
c = parsed_module['WithComments']
assert not c.getComment(), \
'Did not ignore comments in class WithComments (%s)' % c.getComment()
assert not c._docstring, \
'Found unexepcted docstring for class WithComments'
method = c['__init__']
assert not method.getComment(), \
'Did not ignore comments in method WithComments.__init__'
assert not method._docstring, \
'Found unexpected docstring for method WithComments.__init__'
c = parsed_module['WithoutComments']
assert not c.getComment(), \
'Found unexepected comments for class WithoutComments'
assert c._docstring, \
'Did not find docstring for class WithoutComments'
method = c['__init__']
assert not method.getComment(), \
'Found unexpected comments for method WithoutComments.__init__'
assert method._docstring, \
'Did not find docstring for method WithoutComments.__init__'
return
def testDecoratedComments(self):
module = self.parsed_module
assert module['Hashes']._comments == ' \n \n Func with hash lines\n \n \n', \
'Did not find expected comment for Hashes'
assert module['Dashes']._comments == ' \n Func with dash lines\n \n', \
'Did not find expected comment for Dashes'
assert module['Equals']._comments == ' \n Func with equal lines\n \n', \
'Did not find expected comment for Equals'
assert module['Mixed']._comments == ' \n Func with mixed dashes and equals\n \n', \
'Did not find expected comment for Mixed'
expected_for_stt = """
This function has, in the comments about it, a table. That table
should be rendered via STNG to an HTML table in the test output.
|-------------------------------------------------|
| Function | Documentation |
|=================================================|
| '__str__' | This method converts the |
| | the object to a string. |
| | |
| | - Blah |
| | |
| | - Blaf |
| | |
| | |--------------------------| |
| | | Name | Favorite | |
| | | | Color | |
| | |==========================| |
| | | Jim | Red | |
| | |--------------------------| |
| | | John | Blue | |
| | |--------------------------| |
|-------------------------------------------------|
"""
assert module['StructuredTextTable']._comments == expected_for_stt, \
'Did not find expected comment for StructuredTextTable'
return
def testDOSFiles(self):
assert self.parsed_module, 'Did not retrieve any data from %s' % self.filename
assert self.parsed_module['DefaultClassInst']._docstring, \
'Did not get docstring from DefaultClassInst'
assert self.parsed_module['DefaultClassInst']._docstring.find('\r') < 0, \
'Did not strip carriage returns from docstring for DefaultClassInst'
return
def testEmptyModule(self):
assert not self.parsed_module._class_info.items(), 'Found unexpected classes'
assert not self.parsed_module._function_info.items(), 'Found unexpected functions'
return
def testFunctionParameters(self):
m = self.parsed_module
f = m['example_function_with_args']
assert f, 'Did not get function information for example_function_with_args'
expected_parameter_names = ( 'arg1',
'arg2',
'arg3withDefault',
'arg3aWithDefault',
'arg3bWithDefault',
'arg4DefaultInt',
'arg5DefaultTuple',
'arg6DefaultList',
'arg7DefaultNone',
'arg8DefaultName',
'arg9DefaultInstance',
'arg10DefaultInstanceWithParams',
'negativeIntArg',
'floatArg',
'negativeFloatArg',
'mathArg',
'stringArgWithHTML',
)
expected_parameter_info = {
'arg1': (None, None, None),
'arg2': (None, None, None),
'arg3withDefault': (1, "'hi there'", None),
'arg3aWithDefault': (1, '"\'hi again\'"', None),
'arg3bWithDefault': (1, '\'"hi there again"\'', None),
'arg4DefaultInt': (1, '101', None),
'arg5DefaultTuple': (1, '( 1, 2 )', None),
'arg6DefaultList': (1, '[ 3, 4 ]', None),
'arg7DefaultNone': (1, 'None', None),
'arg8DefaultName': (1, 'foo', None),
'arg9DefaultInstance': (1, 'DefaultClassInst()', None),
'arg10DefaultInstanceWithParams': (
1, "DefaultClassInstWithParams(1, 2, ( 'tuple', 'param' ), [ 'list', 'param' ] )", None),
'negativeIntArg': (1, '-1', None),
'floatArg': (1, '1.2', None),
'negativeFloatArg': (1, '-3.4', None),
'mathArg': (1, '1 + 2', None),
'stringArgWithHTML': (1, "'<h1>Hi, Dick & Jane!</h1>'", None),
}
actual_parameter_names = f.getParameterNames()
assert actual_parameter_names, 'Did not get any parameter names.'
assert actual_parameter_names == expected_parameter_names, \
'Actual parameter names (%s) do not match expected (%s)' % \
(actual_parameter_names, expected_parameter_names)
for n in actual_parameter_names:
parameter_info = f.getParameterInfo(n)
assert parameter_info, 'Got no parameter info for %s' % n
try:
expected_info = expected_parameter_info[n]
except KeyError:
self.fail('Unexpected parameter %s found' % n)
assert parameter_info == expected_info, \
'Parameter info for %s does not match expected. %s vs %s' % (
str(parameter_info),
str(expected_info),
)
return
def testNestedClasses(self):
m = self.parsed_module
try:
c = m['OuterClass']
except KeyError:
self.fail('Could not retrieve class "OuterClass"')
assert c.getName() == 'OuterClass', \
'Name of class does not match expected value.'
try:
ic1 = c['InnerClass']
except KeyError:
self.fail('Could not retrieve class "InnerClass" from OuterClass')
else:
assert ic1._docstring == """This class is inside of OuterClass.
This class is nested one level deep.
""", \
'Docstring for InnerClass does not match'
try:
ic2 = ic1['InnerClass2']
except KeyError:
self.fail('Could not retrieve class "InnerClass2" from InnerClass')
else:
assert ic2._docstring == """This class is inside of InnerClass.
This class is nested two levels deep.
""", \
'Docstring for InnerClass2 does not match'
return
def testNestedFunctions(self):
m = self.parsed_module
try:
f = m['OuterFunction']
except KeyError:
self.fail('Could not retrieve function "OuterFunction"')
try:
if1 = f['InnerFunction']
except KeyError:
self.fail('Could not retrieve inner function "InnerFunction"')
else:
assert if1._docstring == "This function is inside of OuterFunction.", \
'Docstring for InnerFunction does not match.'
try:
if2 = if1['InnerFunction2']
except KeyError:
self.fail('Could not retrieve inner function "InnerFunction2"')
else:
assert if2._docstring == "This function is inside of InnerFunction.", \
'Docstring for InnerFunction2 does not match.'
return
def testIncludeCommentsOptionTrue(self):
happydoclib.parseinfo.setOption(include_comments=1)
happydoclib.parseinfo.setOption(useCache=0)
input_filename = os.path.join( 'TestCases',
'parseinfo',
'test_decorated_comments.py'
)
module_info = happydoclib.parseinfo.getDocs(None, input_filename)
func_names = module_info.getFunctionNames()
for func_name in func_names:
func_info = module_info.getFunctionInfo(func_name)
comments = func_info.getComment()
self.failUnless(comments, 'Did not get any comments for %s.' % func_name)
return
def testIncludeCommentsOptionFalse(self):
happydoclib.parseinfo.setOption(include_comments=0)
happydoclib.parseinfo.setOption(useCache=0)
input_filename = os.path.join( 'TestCases',
'parseinfo',
'test_decorated_comments.py'
)
module_info = happydoclib.parseinfo.getDocs(None, input_filename)
func_names = module_info.getFunctionNames()
for func_name in func_names:
func_info = module_info.getFunctionInfo(func_name)
comments = func_info.getComment()
if comments:
self.fail('Got comments for %s.' % func_name)
return
if __name__ == '__main__':
unittest.main()
|