#################################################
# SPYCE - Python-based HTML Scripting
# Copyright (c) 2002 Rimon Barr.
#
# Refer to spyce.py
##################################################
#rimtodo:
# - fix compaction (it assumed newlines parsed independently)
import re # otherwise apache 2.0 pcre library conflicts
# we just can't win! either stack limits (sre), or
# library conflicts (pre)! :)
from cStringIO import StringIO
import sys, string, token, tokenize, os, md5, base64
import spyceTag, spyceUtil, spyce
from spyceException import spyceSyntaxError
__doc__ = '''Compile Spyce files into Python code.'''
##################################################
# Special method names
#
# external interface of generated code
SPYCE_CLASS = 'spyceImpl'
SPYCE_PROCESS_FUNC = 'spyceProcess'
SPYCE_LIBNAME = 'spyceTagcollection'
SPYCE_WRAPPER = '_spyceWrapper' # for raising spyceRuntimeException
# codepoints
GLOBAL_CODEPATH = ['global']
CLASS_CODEPATH = [SPYCE_CLASS]
SPYCEPROCESS_CODEPATH = [SPYCE_PROCESS_FUNC]
# MODULES_CODEPATH is different depending on whether we are compiling a
# tagcollection, and will be created later
TAGLIB_CODEPATH = [SPYCE_PROCESS_FUNC, 'taglibs']
LOGIN_CODEPATH = [SPYCE_PROCESS_FUNC, 'login']
HANDLER_CODEPATH = [SPYCE_PROCESS_FUNC, 'handler']
##################################################
# Dos-to-Unix linebreaks
#
# split a buffer into lines (regardless of terminators)
def splitLines(buf):
lines=[]
f=StringIO(buf)
l=f.readline()
while l:
while l and l[-1] in ['\r', '\n']:
l=l[:-1]
lines.append(l)
l=f.readline()
return lines
# encode document with LF
def CRLF2LF(s):
return string.join(splitLines(s), '\n')+'\n'
# encode document with CRLF
def LF2CRLF(s):
return string.join(splitLines(s), '\r\n')+'\r\n'
##################################################
# Tokens
#
T_ESC = -2
T_EOF = -1
T_TEXT = 0
T_EVAL = 1
T_STMT = 2
T_CHUNK = 3
T_CHUNKC = 4
T_DIRECT = 5
T_LAMBDA = 6
T_END = 7
T_CMNT = 8
T_END_CMNT = 9
TOKENS = (
# in the order that they should be tested
# (i.e. usually longest first)
(T_ESC, r'\\\[\[', r'\\<%', r'\\\]\]', r'\\%>'), # escapes
(T_CHUNKC, r'\[\[!', r'<%!'), # open class chunk
(T_CHUNK, r'\[\[\\', r'<%\\'), # open chunk
(T_EVAL, r'\[\[=', r'<%='), # open eval
(T_DIRECT, r'\[\[\.', r'<%\.', r'<%@'), # open directive
(T_LAMBDA, r'\[\[spy', r'<%spy'), # open lambda
(T_CMNT, r'\[\[--', r'<%--'), # open comment
(T_END_CMNT, r'--\]\]', r'--%>'), # close comment
(T_STMT, r'\[\[', r'<%'), # open statement
(T_END, r'\]\]', r'%>'), # close
)
def genTokensRE(tokens):
regexp = []
typelookup = [None,]
for group in tokens:
type, matchstrings = group[0], group[1:]
for s in matchstrings:
regexp.append('(%s)' % s)
typelookup.append(type)
regexp = string.join(regexp, '|')
return re.compile(regexp, re.M), typelookup
RE_TOKENS = None
TOKEN_TYPES = None
if not RE_TOKENS:
RE_TOKENS, TOKEN_TYPES = genTokensRE(TOKENS)
def spyceTokenize(buf):
# scan using regexp
tokens = []
buflen = len(buf)
pos = 0
brow = bcol = erow = ecol = 0
while pos < buflen:
m = RE_TOKENS.search(buf, pos)
try:
mstart, mend = m.start(), m.end()
other, token = buf[pos:mstart], buf[mstart:mend]
if other:
tokens.append((T_TEXT, other, pos, mstart))
try:
type = TOKEN_TYPES[m.lastindex]
except AttributeError, e:
# Python 1.5 does not support lastindex
lastindex = 1
for x in m.groups():
if x: break
lastindex = lastindex + 1
type = TOKEN_TYPES[lastindex]
if type==T_ESC:
token = token[1:]
type = T_TEXT
tokens.append((type, token, mstart, mend))
pos = mend
except AttributeError, e:
# handle text before EOF...
other = buf[pos:]
if other:
tokens.append((T_TEXT, other, pos, buflen))
pos = buflen
# compute row, col
brow, bcol = 1, 0
tokens2 = []
for type, text, begin, end in tokens:
lines = string.split(text[:-1], '\n')
numlines = len(lines)
erow = brow + numlines - 1
ecol = bcol
if numlines>1: ecol = 0
ecol = ecol + len(lines[-1])
tokens2.append((type, text, (brow, bcol), (erow, ecol)))
if text[-1]=='\n':
brow = erow + 1
bcol = 0
else:
brow = erow
bcol = ecol + 1
return tokens2
def spyceTokenize4Parse(buf):
# add eof and reverse (so that you can pop() tokens)
tokens = spyceTokenize(buf)
try:
_, _, _, end = tokens[-1]
except:
end = 0;
tokens.append((T_EOF, '<EOF>', end, end))
tokens.reverse()
return tokens
def processMagic(buf):
if buf[:2]=='#!':
buf = string.join(string.split(buf, '\n')[1:], '\n')
return buf
##################################################
# Directives / Active Tags / Multi-line quotes
#
DIRECTIVE_NAME = re.compile('[a-zA-Z][-a-zA-Z0-9_:]*')
DIRECTIVE_ATTR = re.compile(
r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*'
r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./:;+*%?!&$\(\)_#=~]*))?')
def parseDirective(text):
"Parse a Spyce directive into name and an attribute list."
attrs = {}
match = DIRECTIVE_NAME.match(text)
if not match: return None, {}
name = string.lower(text[:match.end()])
text = string.strip(text[match.end()+1:])
while text:
match = DIRECTIVE_ATTR.match(text)
if not match: break
attrname, rest, attrvalue = match.group(1, 2, 3)
if not rest: attrvalue = None
elif attrvalue[:1] == "'" == attrvalue[-1:] or \
attrvalue[:1] == '"' == attrvalue[-1:]:
attrvalue = attrvalue[1:-1]
attrs[string.lower(attrname)] = attrvalue
text = text[match.end()+1:]
return name, attrs
RE_LIB_TAG = re.compile(r'''< # beginning of tag
(?P<end>/?) # ending tag
(?P<lib>[a-zA-Z][-.a-zA-Z0-9_]*): # lib name
(?P<name>[a-zA-Z][-.a-zA-Z0-9_]*) # tag name
(?P<attrs>(?:\s+ # attributes
(?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name
(?:\s*=\s* # value indicator
(?:'[^']*' # LITA-enclosed value
|"[^"]*" # LIT-enclosed value
|[^'">\s]+ # bare value
)
)?
)
)*)
\s* # trailing whitespace
(?P<single>/?) # single / unpaired tag
>''', re.VERBOSE) # end of tag
def calcEndPos(begin, str):
if not str: raise 'empty string'
beginrow, begincol = begin
eol = 0
if str[-1]=='\n':
str = str[:-1]+' '
eol = 1
lines = string.split(str, '\n')
endrow = beginrow + len(lines)-1
if endrow!=beginrow:
begincol = 0
endcol = begincol + len(lines[-1]) - 1
beginrow, begincol = endrow, endcol + 1
if eol:
begincol = 0
beginrow = beginrow + 1
return (endrow, endcol), (beginrow, begincol)
def removeMultiLineQuotes(s):
def findMultiLineQuote(s):
quotelist = []
def eatToken(type, string, begin, end, _, quotelist=quotelist):
if type == token.STRING:
if string.startswith('r"""') or string.startswith("r'''") \
or string.startswith('"""') or string.startswith("'''"):
quotelist.append((string, begin, end))
tokenize.tokenize(StringIO(s).readline, eatToken)
return quotelist
def replaceRegionWithLine(s, begin, end, s2):
(beginrow, begincol), (endrow, endcol) = begin, end
beginrow, endrow = beginrow-1, endrow-1
s = string.split(s, '\n')
s1, s3 = s[:beginrow], s[endrow+1:]
s2 = s[beginrow][:begincol] + s2 + s[endrow][endcol:]
return string.join(s1 + [s2] + s3, '\n')
match = findMultiLineQuote(s)
offsets = {}
for _, (obr, _), (oer, _) in match:
offsets[obr] = oer - obr
while match:
s2, begin, end = match[0]
s = replaceRegionWithLine(s, begin, end, `eval(s2)`)
match = findMultiLineQuote(s)
return s, offsets
##################################################
# Pre-Python AST
#
# ast node types
AST_PY = 0
AST_PYEVAL = 1
AST_TEXT = 2
AST_COMPACT = 3
# compacting modes
COMPACT_OFF = 0
COMPACT_LINE = 1
COMPACT_SPACE = 2
COMPACT_FULL = 3
class Codepoint:
def __init__(self, path):
self.elements = {}
self.fragments = []
self.path = path
def add(self, code):
self.fragments.append(code)
if isinstance(code, Codepoint):
self.elements[code.path[-1]] = code
def __str__(self, depth=0):
L = []
indent = '\t' * depth
L.append(indent + 'path: ' + str(self.path))
for code in self.fragments:
if isinstance(code, Codepoint):
L.append(code.__str__(depth + 1))
else:
L.append(indent + str(code))
return '\n'.join(L)
class Leaf:
def __init__(self, type, code, ref):
self.type = type
self.code = code
self.ref = ref
def __str__(self):
return str((self.type, self.code, self.ref))
class ppyAST:
"Generate a pre-Python AST"
def __init__(self):
"Initialise parser data structures, AST, token handlers, ..."
# set up ast
self._root = Codepoint([])
self._activepoint = self._root
self._mods = []
self._taglibs = {}
# routines to navigate AST
def mergeCode(self, codepath):
restore_point = self._activepoint
self.selectCodepath(codepath)
restore_point.fragments += self._activepoint.fragments
self._activepoint.fragments = []
for key in self._activepoint.elements:
restore_point.elements[key] = self._activepoint.elements[key]
self._activepoint.elements = {}
self._activepoint = restore_point
def selectCodepath(self, codepath):
"""
codepath is a list of which key in the elements dict to follow.
selectCodepath starts at the top of the AST and follows this path down.
"""
self._activepoint = self._root
for point in codepath:
self._descendCodepath(point)
def getCodepath(self):
return self._activepoint.path
def _descendCodepath(self, codepoint):
try:
self._activepoint = self._activepoint.elements[codepoint]
except KeyError:
raise 'codepoint %s not found in elements %s of ast %s' % (codepoint, self._activepoint.elements, self._root)
# routines that modify the ast
def appendCodepoint(self, pointname, ref=None, descend=True):
new_path = list(self._activepoint.path)
new_path.append(pointname)
self._activepoint.add(Codepoint(new_path))
if descend:
self._descendCodepath(pointname)
def addCode(self, code, ref=None, path=None):
if path is not None:
restore_point = self.getCodepath()
self.selectCodepath(path)
self._activepoint.add(Leaf(AST_PY, code, ref))
if path is not None:
self.selectCodepath(restore_point)
def addEval(self, eval, ref=None):
self._activepoint.add(Leaf(AST_PYEVAL, eval, ref))
def addCodeIndented(self, code, ref, classcode=0):
code, replacelist = removeMultiLineQuotes(code)
# funky hack: put in NULLs to preserve indentation
# NULLs don't appear in code, and the BraceConverter will
# turn them back into spaces. If we leave them as spaces,
# BraceConverter is just going to ignore them and pay attention
# only to the braces. (not the best compile-time performance!)
code = string.split(code, '\n')
code = map(lambda l: (len(l)-len(string.lstrip(l)), l), code)
code = map(lambda (indent, l): chr(0)*indent + l, code)
code.append('')
# split code lines
(brow, bcol), (erow, ecol), text, file = ref
row = brow
for l in code:
cbcol = 0
cecol = len(l)
if row==brow: cbcol = bcol
if row==erow: cecol = ecol
try: row2 = row + replacelist[row-brow+1]
except: row2 = row
ref = (row, cbcol), (row2, cecol), l, file
if classcode: self.addCode(l, ref, CLASS_CODEPATH)
else: self.addCode(l, ref)
row = row2 + 1
def addText(self, text, ref=None):
self._activepoint.add(Leaf(AST_TEXT, text, ref))
def addCompact(self, compact, ref):
self._activepoint.add(Leaf(AST_COMPACT, compact, ref))
def addModule(self, modname, modfrom, modas):
self._mods.append((modname, modfrom, modas))
##################################################
# Parse
#
class spyceParse:
def initStandard(self, sig):
self._ast.appendCodepoint(GLOBAL_CODEPATH[0])
self._ast.addCode('from spyceException import spyceDone, spyceRedirect, spyceRuntimeException, HandlerError')
self._ast.selectCodepath([])
# class codepoint, for class chunk when we start compiling
# todo: allow this in compiled libraries?
self._ast.appendCodepoint(SPYCE_CLASS)
self._ast.addCode('class %s: {' % (SPYCE_CLASS))
# define spyceProcess
self._ast.selectCodepath([])
self._ast.appendCodepoint(SPYCE_PROCESS_FUNC)
if sig:
sig = 'self, ' + sig
else:
sig = 'self'
self._ast.addCode('def %s(%s): {' % (SPYCE_PROCESS_FUNC, sig))
self._ast.addCode('try:{')
global MODULES_CODEPATH
MODULES_CODEPATH = [SPYCE_PROCESS_FUNC, 'spymod']
self._ast.appendCodepoint(MODULES_CODEPATH[-1], descend=False)
self._ast.appendCodepoint(LOGIN_CODEPATH[-1], descend=False)
self._ast.appendCodepoint(HANDLER_CODEPATH[-1])
self._ast.addCode("_handlers = {}")
self._ast.addCode("_forms_by_handler = {}")
self._ast.selectCodepath(SPYCEPROCESS_CODEPATH)
# set up a point to add taglib load calls later
self._ast.appendCodepoint(TAGLIB_CODEPATH[-1], descend=False)
def finishStandard(self):
# login stuff, must allow login_required to terminate execution before running handlers!
self._ast.selectCodepath(LOGIN_CODEPATH)
# munge login_required tag invocation to the end of this codepath,
# after the request.login_id stuff
old_login_fragments = self._ast._activepoint.fragments
self._ast._activepoint.fragments = []
if self.filename:
# (if no filename, it's a spylamba, and we don't need to re-munge request obj b/c
# it's passed in from a real page request)
self._ast.addCode("import _coreutil")
# login_id() is a callable so that there is no overhead if it's not needed.
# (w/o explicitly binding request in lambda you will get "unknown global 'request'" errors)
self._ast.addCode("request.login_id = lambda request=request, _coreutil=_coreutil: _coreutil.login_from_cookie(request)")
if self._login_possible:
self._ast.addCode("if _coreutil.login_pending(request):{")
self._ast.addCode(" if self._login_validator:{")
self._ast.addCode(" from spyceCompile import _evalWithImport")
self._ast.addCode(" _validator = _evalWithImport(self._login_validator)")
self._ast.addCode(" }else:{")
self._ast.addCode(" import spyceConfig")
self._ast.addCode(" _validator = spyceConfig.login_defaultvalidator")
self._ast.addCode(" }")
self._ast.addCode(" _login_id = _coreutil.login_perform(request, _validator)")
self._ast.addCode(" request.login_id = lambda _login_id=_login_id: _login_id")
self._ast.addCode("}")
if self._login_required:
self._ast.addCode("if not request.login_id():{")
# TODO fix hack of special-casing core taglib here (general case done by addLoadTaglibs)
self._ast.addCode("taglib.load('core','core.py','spy')")
self._ast._activepoint.fragments.extend(old_login_fragments)
self._ast.addCode("}")
# handlers
if self._call_handlers:
self._ast.selectCodepath(HANDLER_CODEPATH)
# call handlers on postback
self._ast.addCode("_validation_error = {}")
self._ast.addCode("for _key in request.getpost():{")
self._ast.addCode(" if _key.startswith('_submit'):{")
self._ast.addCode(" _handlerid = _key[7:]")
# don't assume handler is there -- tag in question could have
# been output by included file or parent, which will run
# its own copy of this code
self._ast.addCode(" if _handlerid in _handlers:{")
self._ast.addCode(" from spyceCompile import _evalWithImport")
self._ast.addCode(" for _handler in _handlers[_handlerid]:{")
# imports done in tag scope won't be visible here; make up for that w/autoimport here
self._ast.addCode(" _f = _evalWithImport(_handler, locals())")
self._ast.addCode(" from spyceCompile import _marshallArgs")
self._ast.addCode(" _args, _kwargs = _marshallArgs(request, _f)")
self._ast.addCode(" try:{")
self._ast.addCode(" _f(*_args, **_kwargs)")
self._ast.addCode(" } except HandlerError, _e:{")
self._ast.addCode(" _form_id = _forms_by_handler[_handlerid]")
self._ast.addCode(" _validation_error[_form_id] = _e")
self._ast.addCode(" }")
self._ast.addCode(" break")
self._ast.addCode("}}}}")
if spyce.DEBUG_ERROR:
self._ast.addCode("if '_handlerid' not in locals():{")
self._ast.addCode(" import spyce; spyce.DEBUG('(no active handlers to run for this form)')")
self._ast.addCode("}")
self._ast.selectCodepath(SPYCEPROCESS_CODEPATH)
# spyceProcess post
self._ast.addCode('} except spyceDone: pass')
self._ast.addCode('except spyceRedirect: raise')
self._ast.addCode('except KeyboardInterrupt: raise')
self._ast.addCode('except:{ raise spyceRuntimeException(%s) }'%SPYCE_WRAPPER)
self._ast.addCode('}}') # matches spyceProcess/SPYCE_CLASS
self._ast.addCode('_has_parent = %r' % self._has_parent, path=CLASS_CODEPATH)
self._ast.addCode('_login_validator = %r' % self._login_validator, path=CLASS_CODEPATH)
if self._taglibs_used:
self._ast.addModule('taglib', None, None)
self._ast.selectCodepath(TAGLIB_CODEPATH)
self.addLoadTaglibs()
if self._load_spylambda: self._ast.addModule('spylambda', None, None)
def initTaglib(self):
global MODULES_CODEPATH
MODULES_CODEPATH = ['spymod']
self._ast.appendCodepoint(MODULES_CODEPATH[-1], descend=False)
pass
def finishTaglib(self):
self._ast.addCode('''
class %s(spyceTagLibrary):
tags = [%s]
''' % (SPYCE_LIBNAME, ','.join(self._tagsdefined)))
def next_tagid(self):
self._tagcount += 1
return self._tagidprefix + str(self._tagcount)
def addLoadTaglibs(self):
for taglib in self._taglibs_used:
libname, libfrom = self._ast._taglibs[taglib]
self._tag_dependencies.append(libfrom)
self._ast.addCode('taglib.load(%s, %s, %s)'%(repr(libname), repr(libfrom), repr(taglib)))
def __init__(self, server, buf, filename, gentaglib, sig):
try:
# initialization
self._current_form_id = None
self._brace_stack = []
self._tagcount = 0
self._server = server
self._tagChecker = spyceTag.spyceTagChecker(server)
self._load_spylambda = False
self._has_parent = False
self._login_required = False
self._login_possible = False
self._login_validator = False
self._call_handlers = False
self._taglibs_used = {}
self._tag_dependencies = []
self._tagsdefined = []
self._definingtag = spyceUtil.attrdict()
self._gentaglib = gentaglib
self._curdir, self._curfile = os.getcwd(), '<string>'
self._tagidprefix = base64.encodestring(md5.md5(buf).digest())[:-1]
self.filename = filename # so request-munger can tell if this is a spylambda (and req is already munged)
if filename:
self._curdir, self._curfile = os.path.split(filename)
if not self._curdir:
self._curdir = os.getcwd() # TODO this is sorta broken; we don't chdir for each request
self._path = os.path.join(self._curdir, self._curfile)
# prime ast
self._ast = ppyAST()
for tag_tuple in server.config.globaltags:
path, name = server._findModule(tag_tuple[0], tag_tuple[1], None)
# a 2.0-style active tag may not call other tags in the same collection
if path == self._path:
continue
args = list(tag_tuple) + [None, None]
self.addTaglib(*args)
if gentaglib:
self.initTaglib()
else:
self.initStandard(sig)
# spyceProcess body
self._tokens = spyceTokenize4Parse(processMagic(buf))
self._tokenType = None
self.popToken()
self.processSpyce()
if self._brace_stack:
ref = self._brace_stack[-1]
raise spyceSyntaxError("unclosed opening brace '{'", ref)
if gentaglib:
self.finishTaglib()
else:
self.finishStandard()
# post processing
self._tagChecker.finish()
except spyceSyntaxError, e:
raise
if e.info:
begin, end, text, _ = e.info
e.info = begin, end, text, self._curfile
raise e
def addTaglib(self, libname, libfrom, libas, fullfile, ref):
if not libas: libas=libname
self._tagChecker.loadLib(libname, libfrom, libas, fullfile, ref)
self._ast._taglibs[libas] = libname, libfrom
def info(self):
return self._ast._root, self._ast._mods, self._tag_dependencies
def popToken(self):
if self._tokenType!=T_EOF:
self._tokenType, self._tokenText, self._tokenBegin, self._tokenEnd = self._tokens.pop()
def processSpyce(self):
while self._tokenType!=T_EOF:
[
self.processText, # T_TEXT
self.processEval, # T_EVAL
self.processStmt, # T_STMT
self.processChunk, # T_CHUNK
self.processClassChunk, # T_CHUNKC
self.processDirective, # T_DIRECT
self.processUnexpected, # T_LAMBDA
self.processUnexpected, # T_END
self.processComment, # T_CMNT
self.processUnexpected, # T_END_CMNT
][self._tokenType]()
self.popToken()
def processComment(self):
# collect comment
self.popToken()
while self._tokenType not in [T_END_CMNT, T_EOF]:
self.popToken()
if self._tokenType==T_EOF:
self.processUnexpected()
def addText(self, text, ref):
if self._gentaglib and not self._definingtag:
s = re.sub(r'''\s''', '', text)
if s:
raise spyceSyntaxError('text found outside of tag definition in tagcollection: "%s"' % s, ref)
return # whitespace is OK, but we don't want to add it to the AST
self._ast.addText(text, ref)
def processText(self):
"Process HTML (possibly with some active tags)"
html, begin, end = self._tokenText, self._tokenBegin, self._tokenEnd
while True:
# TODO refactor this so can share code with include.spycecode
# something like a second-stage parse that separates text into
# PLAIN and TAG sections...
m = RE_LIB_TAG.search(html)
if not m:
break
spyce.DEBUG('active tag hit: %s' % html[m.start():m.end()])
# emit text literal before tag start, if any
plain = html[:m.start()]
if plain:
plain_end, tag_begin = calcEndPos(begin, plain)
self.addText(plain, (begin, plain_end, '<html string>', self._curfile))
else: tag_begin = begin
tag = m.group(0)
tag_end, begin = calcEndPos(tag_begin, tag)
self.processActiveTag(tag,
not not m.group('end'), m.group('lib'), m.group('name'),
m.group('attrs'), not m.group('single'),
tag_begin, tag_end)
html = html[m.end():]
self.addText(html, (begin, end, '<html string>', self._curfile))
def processActiveTag(self, tag, tagend, taglib, tagname, tagattrs, tagpair, begin, end):
"""
Process HTML and Spyce tags
tagend: true if tag starts with </
taglib: tag library alias, e.g. 'spy' or 'f' (as in spy:for, f:checkbox)
tagname: duh
tagattrs: dict of attributes
tagpair: true if tag doesn't end with />
"""
ref = (begin, end, tag, self._curfile)
if self._gentaglib and not self._definingtag:
raise spyceSyntaxError("active tag used outside of tag definition in tagcollection", ref)
# make sure prefix belongs to loaded taglibrary
if not self._ast._taglibs.has_key(taglib):
self.addText(tag, (begin, end, '<html string>', self._curfile))
return
# parse process tag attributes
_, tagattrs = parseDirective('x '+tagattrs)
# get tag class
tagclass = self._tagChecker.getTagClass(self._ast._taglibs[taglib], tagname, ref)
codepath = None
if tagclass.__name__ == 'tag_parent':
self._has_parent = True
elif tagclass.__name__ == 'tag_login_required':
self._login_required = True
codepath = LOGIN_CODEPATH
elif tagclass.__name__ == 'form_form':
if not self._gentaglib:
self._current_form_id = self.next_tagid()
self._ast.addCode("_current_form_id = %r" % self._current_form_id, ref)
if tagclass.__name__ in ['tag_login', 'tag_login_required']:
self._login_possible = True
if 'validator' in tagattrs:
self._login_validator = tagattrs['validator']
if self._gentaglib and self._login_possible:
raise spyceSyntaxError("login tags may not currently be used in user-defined Active Tags")
# syntax check
if not tagend: # start tag
self._tagChecker.startTag(self._ast._taglibs[taglib],
tagname, tagattrs, tagpair, ref)
else: # end tag
self._tagChecker.endTag(self._ast._taglibs[taglib], tagname, ref)
if tag in self._taglibs_used.setdefault(taglib, {}):
firstuse = False
else:
firstuse = True
self._taglibs_used[taglib][tag] = True
if not tagend or not tagpair: # open or singleton tag
if tagclass.handlers is None:
tagid = None
if 'handler' in tagattrs:
raise spyceSyntaxError('''handler cannot apply to this "%s:%s" tag; handlers may only be assigned to tags that have a 'handlers' list, such as the form library's submit tag''' % (taglib, tagname), ref)
else:
tagid = self.next_tagid()
allhandlers = {}
for (subid, subhandlers) in tagclass.handlers.items():
fullid = tagid + subid
allhandlers[fullid] = subhandlers
if 'handler' in tagattrs:
if tagattrs['handler'].startswith('='):
raise spyceSyntaxError("handler attribute may not be evaluated at runtime (i.e., may not start with '=')", ref)
L = tagattrs['handler'].split(',')
if not allhandlers:
allhandlers[tagid] = []
for (fullid, subhandlers) in allhandlers.items():
allhandlers[fullid] = subhandlers + L
# tag doesn't have to do anything further to set up handler,
# but we'll pass the attr along so it can perform any validation it wants
# (e.g, don't allow form action + handler)
elif not allhandlers:
allhandlers[tagid] = []
for (fullid, L) in allhandlers.items():
if not L:
continue
if self._gentaglib:
self._definingtag.handlers[fullid] = L
else:
self._call_handlers = True
self._ast.addCode("_handlers[%r] = %r" % (fullid, L), path=HANDLER_CODEPATH)
self._ast.addCode("_forms_by_handler[%r] = %r" % (fullid, self._current_form_id), path=HANDLER_CODEPATH)
# end handler code
if firstuse:
if tagclass.classcode:
cref = tagclass.classcode
self.addChunk(cref[2], cref, True)
self._ast.addCode('taglib.tagPush(%s, %s, %s, locals(), %s, %s)' % (
repr(taglib), repr(tagname), repr(tagid), repr(tagattrs), repr(tagpair)), ref, codepath)
self._ast.addCode('try: {', ref, codepath)
if tagclass.catches:
self._ast.addCode('try: {', ref, codepath)
if tagclass.conditional:
self._ast.addCode('if taglib.tagBegin(): {', ref, codepath)
else:
self._ast.addCode('taglib.tagBegin()', ref, codepath)
if tagclass.mustend:
self._ast.addCode('try: {', ref, codepath)
if tagclass.loops:
self._ast.addCode('while 1: {', ref, codepath)
# handle exports
if tagclass.exports:
# use _foo instead of __foo to avoid problems with name mangling
# -- since we're in a class, the assignment gets mangled, but the
# code in exec does not, yeilding a NameError. Doh!
self._ast.addCode('_tagexports = taglib.tagExport()', ref, codepath)
self._ast.addCode('for _tagkey in _tagexports: {'
+ """exec("%s = _tagexports['%s']" % (_tagkey, _tagkey))"""
+ '}',
ref, codepath)
if tagend or not tagpair: # close or singleton tag
if tagclass.loops:
self._ast.addCode('if not taglib.tagBody(): break }', ref, codepath)
else:
self._ast.addCode('taglib.tagBody()', ref, codepath)
if tagclass.mustend:
self._ast.addCode('} finally: taglib.tagEnd()', ref, codepath)
else:
self._ast.addCode('taglib.tagEnd()', ref, codepath)
if tagclass.conditional:
self._ast.addCode('}', ref, codepath)
if tagclass.catches:
self._ast.addCode('} except: taglib.tagCatch()', ref, codepath)
self._ast.addCode('} finally: taglib.tagPop()', ref, codepath)
def processEval(self):
# collect expression
begin = self._tokenBegin
self.popToken()
expr = ''
while self._tokenType not in [T_END, T_EOF]:
if self._tokenType==T_TEXT:
expr = expr + self._tokenText
elif self._tokenType==T_LAMBDA:
expr = expr + self.processLambda()
else: self.processUnexpected()
self.popToken()
expr = string.strip(expr)
if not expr: self.processUnexpected()
# add expression to ast
self._ast.addEval(expr, (begin, self._tokenEnd, '='+expr, self._curfile))
def processStmt(self):
# collect statement
self.popToken()
beginrow, begincol = self._tokenBegin
stmt = ''
while self._tokenType not in [T_END, T_EOF]:
if self._tokenType==T_TEXT:
stmt = stmt + self._tokenText
elif self._tokenType==T_LAMBDA:
stmt = stmt + self.processLambda()
else: self.processUnexpected()
endrow, endcol = self._tokenEnd
self.popToken()
if not string.strip(stmt): self.processUnexpected()
# add statement to ast, row-by-row
currow = beginrow
lines = string.split(stmt, '\n')
for l in lines:
if currow==beginrow: curcolbegin = begincol
else: curcolbegin = 0
if currow==endrow: curcolend = endcol
else: curcolend = len(l)
l = string.strip(l)
if l:
ref = ((currow, curcolbegin), (currow, curcolend), l, self._curfile)
def braceTokenEater(type, string, begin, end, line):
if type==token.OP:
if string == '{':
self._brace_stack.append(ref)
elif string == '}':
if not self._brace_stack:
raise spyceSyntaxError("extra close brace '}'", ref)
self._brace_stack.pop()
try:
tokenize.tokenize(StringIO(l).readline, braceTokenEater)
except tokenize.TokenError:
# eof before close brace found; this is expected
pass
self._ast.addCode(l, ref)
currow = currow + 1
def processChunk(self, classChunk=0):
# collect chunk
self.popToken()
begin = self._tokenBegin
chunk = ''
while self._tokenType not in [T_END, T_EOF]:
if self._tokenType==T_TEXT:
chunk = chunk + self._tokenText
elif self._tokenType==T_LAMBDA:
chunk = chunk + self.processLambda()
else: self.processUnexpected()
end = self._tokenEnd
self.popToken()
ref = (begin, end, chunk.strip(), self._curfile)
if self._gentaglib:
if not self._definingtag:
raise spyceSyntaxError('tagcollection code chunks may only appear inside tag definitions', ref )
if classChunk:
self._definingtag.classcode = ref
return
# add chunk block at ast
if chunk:
self.addChunk(chunk, ref, classChunk)
def addChunk(self, chunk, ref, classChunk):
(begin, end, _, curfile) = ref
chunk = string.split(chunk, '\n')
# eliminate initial blank lines
brow, bcol = begin
while chunk and not string.strip(chunk[0]):
chunk = chunk[1:]
brow = brow + 1
bcol = 0
begin = brow, bcol
if not chunk: self.processUnexpected()
# outdent chunk based on first line
# note: modifies multi-line strings having more spaces than first line outdent
# by removing outdent number of spaces at the beginning of each line.
# -- difficult to deal with efficiently (without parsing python) so just
# don't do this!
outdent = len(chunk[0]) - len(string.lstrip(chunk[0]))
for i in range(len(chunk)):
if string.strip(chunk[i][:outdent]):
chunk[i] = ' '*outdent + chunk[i]
chunk = map(lambda l, outdent=outdent: l[outdent:], chunk)
chunk = string.join(chunk, '\n')
ref = (begin, end, chunk, curfile)
try:
self._ast.addCodeIndented(chunk, ref, classChunk)
except tokenize.TokenError, e:
# removeMultiLineQuotes raised
msg, _ = e
raise spyceSyntaxError(msg, ref)
def processClassChunk(self):
self.processChunk(1)
def processDirective(self):
# collect directive
begin = self._tokenBegin
self.popToken()
directive = ''
while self._tokenType not in [T_END, T_EOF]:
if self._tokenType==T_TEXT:
directive = directive + self._tokenText
else: self.processUnexpected()
end = self._tokenEnd
self.popToken()
directive = string.strip(directive)
if not directive: self.processUnexpected()
ref = (begin, end, directive, self._curfile)
# process directives
name, attrs = parseDirective(directive)
if name=='compact':
compact_mode = COMPACT_FULL
if attrs.has_key('mode'):
mode = string.lower(attrs['mode'])
if mode=='off':
compact_mode = COMPACT_OFF
elif mode=='line':
compact_mode = COMPACT_LINE
elif mode=='space':
compact_mode = COMPACT_SPACE
elif mode=='full':
compact_mode = COMPACT_FULL
else:
raise spyceSyntaxError('invalid compacting mode "%s" specified'%mode, ref)
self._ast.addCompact(compact_mode, (begin, end, '<spyce compact directive>', self._curfile))
elif name in ('module', 'import'):
if not attrs.has_key('name') and not attrs.has_key('names'):
raise spyceSyntaxError('name or names attribute required', ref)
if attrs.has_key('names'):
mod_names = filter(None, map(string.strip, string.split(attrs['names'],',')))
for mod_name in mod_names:
self._ast.addModule(mod_name, None, None)
self._ast.addCode('%s.init()'%mod_name, ref)
else:
mod_name = attrs['name']
mod_from = attrs.get('from')
mod_as = attrs.get('as')
mod_args = attrs.get('args', '')
if mod_as: theName=mod_as
else: theName=mod_name
self._ast.addModule(mod_name, mod_from, mod_as)
self._ast.addCode('%s.init(%s)' % (theName, mod_args), ref, MODULES_CODEPATH)
elif name in ('taglib',):
if not attrs.has_key('name') and not attrs.has_key('from'):
raise spyceSyntaxError('at least one of {name, from} attributes required', ref)
taglib_name = attrs.get('name', SPYCE_LIBNAME) # ignored if a tagcollection
taglib_from = attrs.get('from')
taglib_as = attrs.get('as')
path, name = self._server._findModule(taglib_name, taglib_from, self._path)
if path == self._path:
raise spyceSyntaxError('Compiled active tag may not reference other tags in the same library', ref)
self.addTaglib(taglib_name, taglib_from, taglib_as, self._path, ref)
elif name == 'tagcollection':
if not self._gentaglib:
raise spyceSyntaxError('tagcollection directive may only be used in a dedicated tag collection file', ref)
elif name == 'begin':
if not self._gentaglib:
raise spyceSyntaxError('begin directive may only be used in a dedicated tag collection file', ref)
if self._definingtag:
raise spyceSyntaxError('cannot nest begin directives; expected end for "%s" first' % self._definingtag.name, ref)
if not attrs.has_key('name'):
raise spyceSyntaxError('name attribute required', ref)
name = attrs['name']
self._definingtag.name = name
self._definingtag.attrs = []
self._definingtag.handlers = {}
self._definingtag.exports = []
self._definingtag.classcode = None
self._definingtag.buffer = 'buffers' in attrs and eval(attrs['buffers'])
self._tagsdefined.append(name)
self._ast.addCode('class %s(spyceTagPlus):{' % name, ref)
self._ast.addCode("name = '%s'" % name, ref)
self._ast.addCode('buffer = %s' % self._definingtag.buffer, ref)
if 'singleton' in attrs and eval(attrs['singleton']):
if self._definingtag.buffer:
raise spyceSyntaxError('Buffer option is exclusive with singleton option', ref)
self._ast.addCode('def syntax(self):{', ref)
self._ast.addCode('self.syntaxSingleOnly()}', ref)
else:
self._ast.addCode('def syntax(self):{', ref)
self._ast.addCode('self.syntaxPairOnly()}', ref)
self._definingtag.kwattrs = 'kwattrs' in attrs and eval(attrs['kwattrs'])
self._ast.appendCodepoint(name + 'Begin', descend=False)
elif name == 'attr':
if not self._definingtag:
raise spyceSyntaxError('attr found, but no corresponding begin', ref)
if not attrs.has_key('name'):
raise spyceSyntaxError('name attribute required', ref)
name = attrs['name']
if 'default' in attrs:
self._definingtag.attrs.append((name, attrs['default']))
else:
self._definingtag.attrs.append(name)
elif name == 'export':
if not attrs.has_key('var'):
raise spyceSyntaxError('var attribute required', ref)
if self._definingtag.buffer:
raise spyceSyntaxError('buffering tags may not export variables', ref)
if 'as' in attrs:
exportas = attrs['as']
else:
exportas = attrs['var']
self._definingtag.exports.append((attrs['var'], exportas))
elif name == 'end':
if not self._definingtag:
raise spyceSyntaxError('end found, but no corresponding begin', ref)
if self._load_spylambda:
self._ast.addModule('spylambda', None, None)
self._load_spylambda = False
def attrsort(a, b):
return cmp(isinstance(a, tuple), isinstance(b, tuple))
self._definingtag.attrs.sort(attrsort)
L = []
for attr in self._definingtag.attrs:
if attr[0][0] != attr[0]: # not a string, must be a tuple
L.append("%s='%s'" % attr)
else:
L.append(attr)
if self._definingtag.kwattrs:
L.append('**kwargs')
s = ','.join(L)
if s:
s = ',' + s
self._ast.selectCodepath([self._definingtag.name + 'Begin'])
# always emit begin so tagchecker can veryify attrs
self._ast.addCode('def begin(self%s):{' % s, ref)
if self._definingtag.buffer:
self._ast.addCode('pass}', ref)
self._ast.addCode('def body(self, _content):{', ref)
for attr in self._definingtag.attrs:
if attr[0][0] != attr[0]: # not a string, must be a tuple
attr, default = attr
self._ast.addCode("try:{", ref)
self._ast.addCode(" %s = self._attrs['%s']" % (attr, attr), ref)
self._ast.addCode("} except KeyError: {", ref)
self._ast.addCode(" %s = '%s'" % (attr, default), ref)
self._ast.addCode("}", ref)
else:
self._ast.addCode("%s = self._attrs['%s']" % (attr, attr), ref)
for modname in spyce.DEFAULT_MODULES:
self._ast.addCode("%s = self._api.getModule('%s')" % (modname, modname))
for modname, modfrom, modas in self._ast._mods:
self._ast.addCode('%s = self._api._startModule(%s, %s, %s)' % (
modname, repr(modname), repr(modfrom), repr(modas)), ref)
self._ast.mergeCode(MODULES_CODEPATH)
self._ast._mods = []
if self._taglibs_used:
self._ast.addCode("taglib = self._api.getModules()['taglib']", ref)
self.addLoadTaglibs()
self._taglibs_used = {}
if self._definingtag.handlers:
self._ast.addCode("if not self.getParent('form'):{", ref)
self._ast.addCode(" raise 'active handlers may not be used without a parent form active tag, i.e., f:form'}", ref)
self._ast.selectCodepath([])
# record exported values for export() to access
for (var, exportas) in self._definingtag.exports:
self._ast.addCode("self.%s = %s" % (var, var), ref)
self._ast.addCode('}', ref)
if self._definingtag.exports:
self._ast.addCode("exports = 1", ref)
self._ast.addCode("def export(self):{", ref)
L = []
for (var, exportas) in self._definingtag.exports:
L.append("'%s': self.%s" % (exportas, var))
self._ast.addCode("return {%s}" % ','.join(L))
self._ast.addCode('}', ref)
# class code chunk, if any
if self._definingtag.classcode:
self._ast.addCode("classcode = %s" % repr(self._definingtag.classcode), ref)
# handlers
self._ast.addCode("handlers = %s" % repr(self._definingtag.handlers), ref)
self._ast.addCode('}', ref)
self._definingtag = spyceUtil.attrdict()
elif name=='include':
# deprecated (undocumented) post-1.3
if not attrs.has_key('file'):
raise spyceSyntaxError('file attribute missing', ref)
filename = spyceUtil.url2file(attrs['file'], os.path.join(self._curdir, self._curfile))
f = None
try:
try:
f = open(filename)
buf = f.read()
finally:
if f: f.close()
except KeyboardInterrupt: raise
except:
raise spyceSyntaxError('unable to open included file: %s'%filename, ref)
prev = (self._curdir, self._curfile, self._tokens,
self._tokenType, self._tokenText, self._tokenBegin, self._tokenEnd)
self._curdir, self._curfile = os.path.dirname(filename), filename
self._tokens = spyceTokenize4Parse(processMagic(buf))
self.popToken()
self.processSpyce()
(self._curdir, self._curfile, self._tokens,
self._tokenType, self._tokenText, self._tokenBegin, self._tokenEnd) = prev
else:
raise spyceSyntaxError('invalid spyce directive', ref)
def processLambda(self):
# collect lambda
self.popToken()
begin = self._tokenBegin
lamb = ''
depth = 1
while self._tokenType!=T_EOF:
if self._tokenType in [T_END,]:
depth = depth - 1
if not depth: break
lamb = lamb + self._tokenText
elif self._tokenType in [T_EVAL, T_STMT, T_CHUNK, T_CHUNKC, T_DIRECT, T_LAMBDA]:
depth = depth + 1
lamb = lamb + self._tokenText
elif self._tokenType==T_CMNT:
self.processComment()
else:
lamb = lamb + self._tokenText
end = self._tokenEnd
self.popToken()
# process lambda
lamb = string.split(lamb, ':')
try:
params = lamb[0]
memoize = 0
if params and params[0]=='!':
params = params[1:]
memoize = 1
lamb = string.join(lamb[1:],':')
except:
raise spyceSyntaxError('invalid spyce lambda', (begin, end, lamb, self._curfile))
self._load_spylambda = True
lamb = 'spylambda.define(%s,%s,%d)' % (`string.strip(params)`, `lamb`, memoize)
return lamb
def processUnexpected(self):
raise spyceSyntaxError('unexpected token: "%s"'%self._tokenText,
(self._tokenBegin, self._tokenEnd, self._tokenText, self._curfile))
##################################################
# Peep-hole optimizer
#
class spyceOptimize:
def __init__(self, ast):
self.compaction(ast)
self.sideBySideWrites(ast)
#self.splitCodeLines(ast)
def splitCodeLines(self, ast):
nodes = ast.fragments
i = 0
while i < len(nodes):
row = 1
if isinstance(nodes[i], Codepoint):
self.splitCodeLines(nodes[i])
elif nodes[i].type == AST_PY and nodes[i].ref:
code = nodes[i].code
(brow, bcol), (erow, ecol), code, file = nodes[i].ref
lines = string.split(code, '\n')
if code==text and len(lines)>1:
del nodes[i]
row = brow
for l in lines:
cbcol = 0
cecol = len(l)
if row==brow: cbcol = bcol
if row==erow: becol = ecol
nodes.insert(i+(brow-row), Leaf(AST_PY, l, ((row, cbcol), (row, cecol), l, file)))
row = row + 1
i = i + row
def sideBySideWrites(self, ast):
nodes = ast.fragments
i = 0
while i < len(nodes):
if isinstance(nodes[i], Codepoint):
self.sideBySideWrites(nodes[i])
elif i + 1 < len(nodes) and isinstance(nodes[i + 1], Leaf):
type1, text1, ref1 = (nodes[i].type, nodes[i].code, nodes[i].ref)
type2, text2, ref2 = (nodes[i + 1].type, nodes[i + 1].code, nodes[i + 1].ref)
file1 = None
file2 = None
if ref1:
_, _, _, file1 = ref1
if ref2:
_, _, _, file2 = ref2
if type1==AST_TEXT and type2==AST_TEXT and file1==file2:
text = text1 + text2
begin, _, orig, _ = ref1
_, end, _, _ = ref2
nodes[i] = Leaf(AST_TEXT, text, (begin, end, orig, file1))
del nodes[i+1]
i -= 1
i += 1
def compaction(self, ast):
nodes = ast.fragments
compact = COMPACT_LINE
i = 0
while i < len(nodes):
if isinstance(nodes[i], Codepoint):
self.compaction(nodes[i])
else:
type, text, ref = (nodes[i].type, nodes[i].code, nodes[i].ref)
if type==AST_COMPACT:
compact = text
elif type==AST_TEXT:
# line compaction
if compact==COMPACT_LINE or compact==COMPACT_FULL:
# remove any trailing whitespace
text = string.split(text, '\n')
for j in range(len(text)-1):
text[j] = string.rstrip(text[j])
text = string.join(text, '\n')
# gobble the end of the line
((row, _), _, _, file) = ref
rowtext = string.split(text, '\n')
if rowtext: rowtext = string.strip(rowtext[0])
crow = row ; cfile = file
j = i - 1
while j > 0 and not rowtext and isinstance(nodes[j], Leaf):
type2, text2, ref2 = (nodes[j].type, nodes[j].code, nodes[j].ref)
if ref2: (_, (crow, _), _, cfile) = ref2
if crow != row or file != cfile: break
if type2 == AST_TEXT:
text2 = string.split(text2, '\n')
if text2: text2 = text2[-1]
rowtext += string.strip(text2)
elif type2 == AST_PYEVAL:
rowtext = 'x'
j -= 1
if not rowtext:
text = string.split(text, '\n')
if text and not string.strip(text[0]):
text = text[1:]
text = string.join(text, '\n')
# gobble beginning of the line
(_, (row, _), _, file) = ref
rowtext = string.split(text, '\n')
if rowtext: rowtext = string.strip(rowtext[-1])
crow = row ; cfile = file
j = i + 1
while j < len(nodes) and not rowtext and isinstance(nodes[j], Leaf):
type2, text2, ref2 = (nodes[j].type, nodes[j].code, nodes[j].ref)
if ref2: ((crow, _), _, _, cfile) = ref2
if crow != row or file != cfile: break
if type2 == AST_TEXT:
text2 = string.split(text2, '\n')
if text2: text2 = text2[0]
rowtext += string.strip(text2)
elif type2 == AST_PYEVAL:
rowtext = 'x'
j += 1
if not rowtext:
text = string.split(text, '\n')
if text: text[-1] = string.strip(text[-1])
text = string.join(text, '\n')
# space compaction
if compact==COMPACT_SPACE or compact==COMPACT_FULL:
text = spyceUtil.spaceCompact(text)
# update text, if any
if text: nodes[i] = Leaf(type, text, ref)
else:
del nodes[i]
i -= 1
elif type in [AST_PY, AST_PYEVAL, None]:
pass
else:
raise 'error: unknown AST node type'
i = i + 1
##################################################
# Output classes
#
class LineWriter:
"Output class that counts lines written."
def __init__(self, f, initialLine = 1):
self.f = f
self.lineno = initialLine
def write(self, s):
self.f.write(s)
self.lineno = self.lineno + len(string.split(s,'\n'))-1
def writeln(self, s):
self.f.write(s+'\n')
def close(self):
self.f.close()
class IndentingWriter:
"Output class that helps with indentation of code."
# Note: this writer is line-oriented.
def __init__(self, f, indentSize=2):
self._f = f
self._indentSize = indentSize
self._indent = 0
self._indentString = ' '*(self._indent*self._indentSize)
self._currentLine = ''
def close(self):
if self._indent > 0:
raise 'unmatched open brace'
self._f.close()
def indent(self):
self._indent = self._indent + 1
self._indentString = ' '*(self._indent*self._indentSize)
def outdent(self):
self._indent = self._indent - 1
if self._indent<0:
raise 'unmatched close brace'
self._indentString = ' '*(self._indent*self._indentSize)
def dumpLine(self, s):
self._f.write(self._indentString+s+'\n')
def write(self, s):
self._currentLine = self._currentLine + s
lines = string.split(self._currentLine, '\n')
for l in lines[:-1]:
self.dumpLine(l)
self._currentLine=lines[-1]
def writeln(self, s=''):
self.write(s+'\n')
# remaining methods are defined in terms of writeln(), indent(), outdent()
def pln(self, s=''):
self.writeln(s)
def pIln(self, s=''):
self.indent(); self.pln(s)
def plnI(self, s=''):
self.pln(s); self.indent()
def pOln(self, s=''):
self.outdent(); self.pln(s)
def plnO(self, s=''):
self.pln(s); self.outdent()
def pOlnI(self, s=''):
self.outdent(); self.pln(s); self.indent()
def pIlnO(self, s=''):
self.indent(); self.pln(s); self.outdent()
##################################################
# Print out Braced Python
#
class emitBracedPython:
def __init__(self, out, ast, gentaglib):
self._gentaglib = gentaglib
out = LineWriter(out)
self._spyceRefs = {}
# text compaction
self.compact = COMPACT_LINE
self._gobblelineNumber = 1
self._gobblelineText = ''
# do the deed!
self.emitSpyceRec(out, ast)
def getSpyceRefs(self):
return self._spyceRefs
def emitSpyceRec(self, out, ast):
nodes = ast.fragments
if self._gentaglib:
outstr = 'self._out'
else:
outstr = 'response'
for code in nodes:
if isinstance(code, Codepoint):
self.emitSpyceRec(out, code)
continue
type, text, ref = (code.type, code.code, code.ref)
line1 = out.lineno
if type==AST_TEXT:
out.write('%s.writeStatic(%s)\n' % (outstr, `text`))
elif type==AST_PY:
out.write(text+'\n')
elif type==AST_PYEVAL:
out.write('%s.writeExpr(%s)\n' % (outstr, text))
elif type==AST_COMPACT:
self.compact = text
else:
raise 'error: unknown AST node type'
line2 = out.lineno
if ref:
for l in range(line1, line2):
self._spyceRefs[l] = ref
if not nodes and not ast.elements:
out.write('pass\n')
##################################################
# Print out regular Python
#
class BraceConverter:
"Convert Python with braces into indented (normal) Python code."
def __init__(self, out):
self.out = IndentingWriter(out)
self.prevname = 0
self.prevstring = 0
self.dictlevel = 0
def emitToken(self, type, string):
if type==token.NAME:
if self.prevname: self.out.write(' ')
if self.prevstring: self.out.write(' ')
self.out.write(string)
elif type==token.STRING:
if self.prevname: self.out.write(' ')
string = `eval(string)` # get rid of multi-line strings
self.out.write(string)
elif type==token.NUMBER:
if self.prevname: self.out.write(' ')
self.out.write(string)
elif type==token.OP:
if string=='{':
if self.prevcolon and not self.dictlevel:
self.out.plnI()
else:
self.dictlevel = self.dictlevel + 1
self.out.write(string)
elif string=='}':
if not self.dictlevel:
self.out.plnO()
else:
self.dictlevel = self.dictlevel - 1
self.out.write(string)
else:
self.out.write(string)
elif type==token.ERRORTOKEN and string==chr(0):
self.out.write(' ')
else:
self.out.write(string)
self.prevname = type==token.NAME
self.prevstring = type==token.STRING
self.prevcolon = type==token.OP and string==':'
def emitPython(out, bracedPythonCode, spyceRefs):
out = LineWriter(out)
spyceRefs2 = {}
braceConv = BraceConverter(out)
def eatToken(type, string, begin, end, _, out=out, braceConv=braceConv, spyceRefs=spyceRefs, spyceRefs2=spyceRefs2):
try:
beginrow, _ = begin
line1 = out.lineno
try:
braceConv.emitToken(type, string)
except:
raise spyceSyntaxError('emitToken %s: %s' % (string, spyceUtil.exceptionString()))
line2 = out.lineno
if spyceRefs.has_key(beginrow):
for l in range(line1, line2):
spyceRefs2[l] = spyceRefs[beginrow]
except:
raise spyceSyntaxError('eatToken: %s' % spyceUtil.exceptionString())
try:
tokenize.tokenize(StringIO(bracedPythonCode).readline, eatToken)
except tokenize.TokenError, e:
msg, (row, col) = e
raise spyceSyntaxError('tokenization error "%s" at (%d, %d) in\n%s'
% (msg, row, col, bracedPythonCode))
return spyceRefs2
def calcRowCol(str, pos):
lines = string.split(str, '\n')
row = 1
while pos > len(lines[0]):
pos = pos - len(lines[0]) - 1
del lines[0]
row = row + 1
return row, pos
_bool_values = {
'true': True,
'false': False,
't': True,
'f': False,
'yes': True,
'no': False,
'on': True,
'off': False,
}
def _convertArg(v, convert_to):
if convert_to == 'int':
try:
v = int(v)
except ValueError:
raise ValueError('invalid %s: %s' % (convert_to, v))
elif convert_to == 'float':
try:
v = float(v)
except ValueError:
raise ValueError('invalid %s: %s' % (convert_to, v))
elif convert_to == 'bool':
if v:
try:
v = int(v)
except ValueError:
# (using eval would be a security hole)
try:
v = _bool_values[v.lower()]
except KeyError:
raise ValueError('invalid %s: %s' % (convert_to, v))
else:
# int succeeded, now make bool
v = bool(v)
else:
# empty string is always False
v = False
return v
def _evalWithImport(expr, env=None):
result = None
L = expr.split('.')
if len(L) > 1:
prefix = L[0]
try:
eval(prefix, env)
except NameError:
try:
code = "%s = __import__('%s')" % (prefix, prefix)
d = {}
# an "unqualified exec" causes strange errors in user code;
# see http://spyced.blogspot.com/2005/04/how-well-do-you-know-python-part-4.html
exec code in d
result = eval(expr, d)
except ImportError:
import spyceConfig, spyceUtil
msg = '''Unable to import %s while trying to execute %s.
You probably need to add its location to sys.path in your spyce config file
(%s)
Raw error message was: %s''' % (prefix, expr, spyceConfig.__file__, spyceUtil.exceptionString())
if not result:
result = eval(expr, env)
return result
# called by generated code
def _marshallArgs(req, callable):
import inspect
(desired_args, _, _, defaults) = inspect.getargspec(callable)
if defaults:
args_with_defaults = dict(zip(desired_args[-len(defaults):], defaults))
else:
args_with_defaults = {}
# ignore 'self'
if desired_args[0] == 'self':
desired_args = desired_args[1:]
if not desired_args:
return []
# pre-process request input
input = {}
for rawname, values in req.getpost().iteritems():
L = rawname.split(':')
name, modifiers = L[0], L[1:]
op = 'assign'
convert = ''
for m in modifiers:
if m == 'list':
op = 'append'
elif m == 'int':
convert = 'int'
elif m == 'float':
convert = 'float'
elif m == 'bool':
convert = 'bool'
else:
raise 'invalid argument transformation %s' % m
if op == 'assign':
input[name] = _convertArg(values[0], convert)
else:
input[name] = []
for v in values:
input[name].append(_convertArg(v, convert))
spyce.DEBUG('marshalled input is %s' % input)
# first non-self arg is modulefinder; others we try to look up in request
from spyceModule import moduleFinder
args = [moduleFinder(req._api)]
kwargs = {}
for argname in desired_args[1:]:
try:
v = input[argname]
except KeyError:
if argname in args_with_defaults:
continue
raise 'Required parameter %s not present in request' % argname
if argname in args_with_defaults:
kwargs[argname] = v
else:
args.append(v)
return args, kwargs
##############################################
# Compile spyce files
#
# (sig is usually '', but spylambda sticks its arguments there; spy:parent puts child there too)
def spyceCompile(buf, filename, sig, server, gentaglib=False):
# parse
ast, libs, tags = spyceParse(server, CRLF2LF(buf), filename, gentaglib, sig).info()
# optimize the ast
spyceOptimize(ast)
# generate braced code
out = StringIO()
refs = emitBracedPython(out, ast, gentaglib).getSpyceRefs()
# then, generate regular python code
bracedPython = out.getvalue()
out = StringIO()
refs = emitPython(out, bracedPython, refs)
return out.getvalue(), refs, libs, tags
def test():
import spyce
f = open(sys.argv[1])
gentaglib = spyceUtil.isTagCollection(f)
spycecode = f.read()
f.close()
tokens = spyceTokenize(processMagic(CRLF2LF(spycecode)))
print 'TOKENS:'
for type, text, begin, end in tokens:
print '%s (%s, %s): %s' % (type, begin, end, `text`)
pythoncode, refs, libs, tags = spyceCompile(spycecode, sys.argv[1], '', spyce.getServer(), gentaglib)
L = pythoncode.split('\n')
print 'CODE:'
for i in range(len(L)):
print '%s %s' % (str(i + 1).rjust(3), L[i])
print 'REFS:'
for line, ref in refs.items():
print '%s %s' % (str(line).rjust(3), ref)
print 'REFERENCED MODULES: %s' % libs
if __name__ == '__main__':
test()
|