# rules for python
# based on grammar given in Programming Python by Mark Lutz
# EDIT THIS: THE DIRECTORY IN WHICH TO MARSHAL THE
# GRAMMAR DATA STRUCTURES.
#
ARCHIVE = "."
marshalfilename = ARCHIVE + "/pygram.mar"
pyrules = """
all ::
## input terminates with "fake" dedent (forces read of all file)
@R all1 :: all >> file_input DEDENT
## 1 term newline
##@R lead_blank :: file_input >> NEWLINE file_input
@R top_stmt :: file_input >> file_input stmt
@R file_input :: file_input >> stmt
## 2
@R simple :: stmt >> simple_stmt
@R compound :: stmt >> compound_stmt
## 3 punct ; term NEWLINE
@R one_small :: simple_stmt >> small_stmt NEWLINE
@R more_small :: simple_stmt >> small_stmt ; simple_stmt
@R small_semi :: simple_stmt >> small_stmt ; NEWLINE
## 4 kw pass
@R smexpr :: small_stmt >> expr_stmt
@R smassn :: small_stmt >> assn
@R smprint :: small_stmt >> print_stmt
@R smdel :: small_stmt >> del_stmt
@R smpass :: small_stmt >> pass
@R smflow :: small_stmt >> flow_stmt
@R smimport :: small_stmt >> import_stmt
@R smglobal :: small_stmt >> global_stmt
## access ignored
@R smexec :: small_stmt >> exec_stmt
## 5
@R cmif :: compound_stmt >> if_stmt
@R cmwhile :: compound_stmt >> while_stmt
@R cmfor :: compound_stmt >> for_stmt
@R cmtry :: compound_stmt >> try_stmt
@R cmdef :: compound_stmt >> funcdef
@R cmclass :: compound_stmt >> classdef
##6
@R exprlist :: expr_stmt >> testlist
##@R assignment :: expr_stmt >> assn
@R assn1 :: assn >> testlist = testlist
@R assnn :: assn >> testlist = assn
@R assn1c :: assn >> testlist , = testlist
@R assn1c2 :: assn >> testlist , = testlist ,
@R assnnc :: assn >> testlist , = assn
##testing @R exprassn :: expr_stmt >> expr_stmt = testlist
@R exprlistc :: expr_stmt >> testlist ,
##testing @R exprassnc :: expr_stmt >> expr_stmt = testlist ,
##7 kw print
@R rprint0 :: print_stmt >> print
@R rprint :: print_stmt >> print testlist
@R rprintc :: print_stmt >> print testlist ,
##8 kw del
@R rdel :: del_stmt >> del exprlist
##9 trivially handled in #4
##10 kw raise continue break return
## eliminates 11 12 13 14
@R rbreak :: flow_stmt >> break
@R rcontinue :: flow_stmt >> continue
@R rreturn0 :: flow_stmt >> return
@R rreturn :: flow_stmt >> return testlist
@R rreturnc :: flow_stmt >> return testlist ,
@R rraise1 :: flow_stmt >> raise test
@R rraise2 :: flow_stmt >> raise test , test
@R rraise3 :: flow_stmt >> raise test , test , test
## 11 12 13 14 skipped
## 15 kw import from
@R rimport :: import_stmt >> import dotted_name_list
@R rimportc :: import_stmt >> import dotted_name_list ,
@R dnlist1 :: dotted_name_list >> dotted_name
@R dnlistn :: dotted_name_list >> dotted_name_list , dotted_name
@R rfrom :: import_stmt >> from dotted_name import name_list
@R rfroms :: import_stmt >> from dotted_name import *
@R rfromc :: import_stmt >> from dotted_name import name_list ,
@R nlistn :: name_list >> name_list , NAME
@R nlist1 :: name_list >> NAME
##16 nt NAME
@R dn1 :: dotted_name >> NAME
@R dnn :: dotted_name >> dotted_name . NAME
##17 kw global
@R global1 :: global_stmt >> global NAME
@R globaln :: global_stmt >> global_stmt , NAME
## 18 19 ignored
##20 kw exec in
@R exec1 :: exec_stmt >> exec expr
@R exec2 :: exec_stmt >> exec expr in test
@R exec3 :: exec_stmt >> exec expr in test , test
##21 kw if elif else punct :
@R ifr :: if_stmt >> if test : suite elifs
@R elifs0 :: elifs >>
@R relse :: elifs >> else : suite
@R elifsn :: elifs >> elif test : suite elifs
##22 kw while
@R while1 :: while_stmt >>
while test :
suite
@R while2 :: while_stmt >>
while test :
suite
else :
suite
##23 kw for
@R for1 :: for_stmt >>
for exprlist in testlist :
suite
@R for2 :: for_stmt >>
for exprlist in testlist :
suite
else :
suite
##24 kw try
@R tryr :: try_stmt >> try : suite excepts
@R excepts1 :: excepts >> except_clause : suite
@R excepts2 :: excepts >> except_clause : suite else : suite
@R exceptsn :: excepts >> except_clause : suite excepts
@R tryf :: try_stmt >> try : suite finally : suite
##25 kw except
@R except0 :: except_clause >> except
@R except1 :: except_clause >> except test
@R except2 :: except_clause >> except test , test
##26
@R class1 :: classdef >> class NAME : suite
@R class2 :: classdef >> class NAME ( testlist ) : suite
##27 kw def
@R rdef :: funcdef >> def NAME parameters : suite
##28, 29 punct = *
## (modified from grammar presented)
@R params1 :: parameters >> ( varargslist )
@R params1c :: parameters >> ( varargslist , )
@R params2 :: varargslist >>
## this is way too permissive: fix at semantic level
@R params3 :: varargslist >> arg
@R params4 :: varargslist >> varargslist , arg
@R argd :: arg >> NAME = test
@R arg2 :: arg >> fpdef
@R arg3 :: arg >> * NAME
@R arg4 :: arg >> ** NAME
## 30
@R fpdef1 :: fpdef >> NAME
@R fpdef2 :: fpdef >> ( fplist )
@R fpdef2c :: fpdef >> ( fplist , )
##31
@R fplist1 :: fplist >> fpdef
@R fplistn :: fplist >> fplist , fpdef
##32 t INDENT DEDENT
@R ssuite :: suite >> simple_stmt
@R csuite :: suite >> NEWLINE INDENT stmtseq DEDENT
@R stmtseq1 :: stmtseq >> stmt
@R stmtseqn :: stmtseq >> stmtseq stmt
##33 kw or cancels 53
@R testor :: test >> or_test
@R testand :: or_test >> and_test
@R testor1 :: or_test >> or_test or and_test
## @R testlambda0 :: test >> lambda : test REDUNDANT
@R testlambda1 :: test >> lambda varargslist : test
##34 kw and
@R andnot :: and_test >> not_test
@R andand :: and_test >> and_test and not_test
##35 kw not
@R notnot :: not_test >> not not_test
@R notcmp :: not_test >> comparison
##36 NOTE KWS == >= <= <> !=
@R cmpexpr :: comparison >> expr
@R cmplt :: comparison >> comparison < expr
@R cmpgt :: comparison >> comparison > expr
@R cmpeq :: comparison >> comparison == expr
@R cmpge :: comparison >> comparison >= expr
@R cmple :: comparison >> comparison <= expr
@R cmpnep :: comparison >> comparison <> expr
@R cmpne :: comparison >> comparison != expr
@R cmpin :: comparison >> comparison in expr
@R cmpnotin :: comparison >> comparison not in expr
@R cmpis :: comparison >> comparison is expr
@R cmpisnot :: comparison >> comparison is not expr
##37 kw is not punct > < ! (eliminated)
##38 p |
@R expr_xor :: expr >> xor_expr
@R expr_lor :: expr >> expr | xor_expr
##39 p ^
@R xor_and :: xor_expr >> and_expr
@R xor_xor :: xor_expr >> xor_expr ^ and_expr
##40
@R and_shift :: and_expr >> shift_expr
@R and_and :: and_expr >> and_expr & shift_expr
##41 note kw's << >x> note goofy x to avoid confusing the grammar
@R shift_arith :: shift_expr >> arith_expr
@R shift_left :: shift_expr >> shift_expr << arith_expr
@R shift_right :: shift_expr >> shift_expr >x> arith_expr
##42
@R arith_term :: arith_expr >> term
@R arith_plus :: arith_expr >> arith_expr + term
@R arith_minus :: arith_expr >> arith_expr - term
##43 p */%
@R termfactor :: term >> factor
@R termmul :: term >> term * factor
@R termdiv :: term >> term / factor
@R termmod :: term >> term % factor
## stuff for power
@R factorpower :: factor >> power
@R factorexp :: factor >> factor ** power
##44 p ~
@R powera :: power >> atom trailerlist
@R trailerlist0 :: trailerlist >>
@R trailerlistn :: trailerlist >> trailer trailerlist
@R powerp :: power >> + power
@R powerm :: power >> - power
@R poweri :: power >> ~ power
##45 t NUMBER STRING
@R nulltup :: atom >> ( )
@R parens :: atom >> ( testlist )
@R parensc :: atom >> ( testlist , )
@R nulllist :: atom >> [ ]
@R list :: atom >> [ testlist ]
@R listc :: atom >> [ testlist , ]
@R nulldict :: atom >> { }
@R dict :: atom >> { dictmaker }
@R dictc :: atom >> { dictmaker , }
@R repr :: atom >> ` testlist `
## @R reprc :: atom >> ` testlist , ` doesn't work, apparently
@R aname :: atom >> NAME
## note number to be broken out into FLOAT OCTINT HEXINT INT
@R anumber :: atom >> NUMBER
@R astring :: atom >> stringseq
@R stringseq0 :: stringseq >> STRING
@R stringseqn :: stringseq >> stringseq STRING
##46
@R nullcall :: trailer >> ( )
@R call :: trailer >> ( arglist )
@R callc :: trailer >> ( arglist , )
@R index :: trailer >> [ subscriptdots ]
@R getattr :: trailer >> . NAME
##47
@R arg1 :: arglist >> argument
@R argn :: arglist >> arglist , argument
##@R argn1 :: arglist >> arglist , NAME = test
##48 ( !!!! is this wrong in PP?)
@R posarg :: argument >> test
## here the left test should be a NAME always, but parser doesn't like it
@R namearg :: argument >> test = test
##49 this IS wrong in PP (numeric ext)
@R nodots :: subscriptdots >> subscriptseq
@R yesdots :: subscriptdots >> subscriptseq , . . . , subscriptseq
@R subscript1 :: subscriptseq >> subscript
@R subscriptn :: subscriptseq >> subscriptseq , subscript
@R subscriptt :: subscript >> test
@R subscripts0 :: subscript >> :
@R subscriptsL :: subscript >> test :
@R subscriptsR :: subscript >> : test
@R subscripts :: subscript >> test : test
##50
@R exprlist1 :: exprlist >> expr
@R exprlistn :: exprlist >> exprlist , expr
##51
@R testlist0 :: testlist >> test
@R testlistn :: testlist >> testlist , test
##52
@R dictmaker1 :: dictmaker >> test : test
@R dictmaker2 :: dictmaker >> dictmaker , test : test
"""
nonterms = """
subscriptdots subscript arg
argument arglist subscriptseq params trailerlist
factor atom trailer dictmaker stringseq power
xor_expr and_expr shift_expr arith_expr term
and_test or_test not_test comparison comp_op expr
fplist stmtseq varargslist assn
expr elifs suite excepts parameters pbasic pdefault pspecial
testlist exprlist test dotted_name_list dotted_name name_list
if_stmt while_stmt for_stmt try_stmt funcdef classdef
expr_stmt print_stmt del_stmt flow_stmt import_stmt global_stmt
small_stmt compound_stmt stmt simple_stmt exec_stmt
file_input except_clause fpdef cmp_op
all
"""
import string
# python needs special handling for the lexical stuff
NAMEre = "[" + string.letters + "_][" + string.letters+string.digits +"]*"
NUMBERre = "[" + string.digits + "]+" # temporary!
STRINGre = '"[^"\n]*"' # to be overridden in lexdict
#NEWLINEre = "\n" # to be overridden in lexdict
INDENTre = "#" # a fake! to be overridden
DEDENTre = "#" # a fake! to be overridden
def echo(str):
return str
def DeclareTerminals(Grammar):
Grammar.Addterm("NAME", NAMEre, echo)
Grammar.Addterm("NUMBER", NUMBERre, echo)
Grammar.Addterm("STRING", STRINGre, echo)
#Grammar.Addterm("NEWLINE", NEWLINEre, echo) # newline is kw!
Grammar.Addterm("INDENT", INDENTre, echo)
Grammar.Addterm("DEDENT", DEDENTre, echo)
# not >x> is a fake!
keywords = """
and break class continue def del elif else except exec
finally for from global if import in is lambda not or pass
print raise return try while == >= <= <> != >x> << NEWLINE
**
"""
import kjParser, string, re
from kjParser import KEYFLAG,ENDOFFILETERM
alphanumunder = string.letters+string.digits+"_"
alpha = string.letters + "_"
# components that are part of a identifier (cannot be next to kw).
id_letters = map(None, alphanumunder)
# terminator re for names
nametermre = "[^" + alphanumunder + "]"
nameterm = re.compile(nametermre)
# terminator re for numbers (same as above but allow "." in num).
numtermre = "[^" + alphanumunder + "\.]"
numterm = re.compile(numtermre)
parseerror = "parseerror"
pycommentre = r"(#.*)"
# whitespace regex outside of brackets
# white followed by (comment\n maybe repeated)
# DON'T EAT NEWLINE!!
pywhiteoutre = r"([ \t\r\014]|[\]\n)*%s?" % pycommentre
pywhiteout = re.compile(pywhiteoutre)
# whitespace regex inside brackets
# white or newline possibly followed by comment, all maybe repeated
pywhiteinre = pywhiteoutre #"[ \t\r]*(\\\\\n)*%s?" % pycommentre
pywhitein = re.compile(pywhiteinre)
# totally blank lines (only recognize if next char is newline)
#allblankre = "\n" + pywhiteinre
#allblank = re.compile(allblankre)
# re for indentation (might accept empty string)
indentp = re.compile(r"[\t ]*")
# two char kws and puncts
char2kw = ["if", "or", "in", "is"]
punct2 = ["<>", "<<", ">>", "<=", ">=", "!=", "**", "=="]
# >two char kws as map of first 3 chars to others
char3k_data = """
and break class continue def del elif else except
finally for from global import lambda not pass print
raise return try while exec
"""
char3kw = string.split(char3k_data)
char3kwdict = {}
for x in char3kw:
char3kwdict[x[:3]] = x
# NOTE: newline is treated same as a punctuation
# NOTE: "' ARE NOT PUNCTS
punct = "~!#%^&*()-+=|{}<>,.;:/[]{}\n`"
punctlist = map(None, punct)
kwmap = {}
for x in char2kw + punct2 + char3kw + map(None, punct):
# everything parses as length 1 to the outer world.
kwmap[x] = (((KEYFLAG, x), x), 1)
# special hack
kwmap[">>"] = (((KEYFLAG, ">x>"), ">x>"), 1)
newlineresult = kwmap["\n"] = (((KEYFLAG, "NEWLINE"), "NEWLINE"), 1)
#finaldedent = (((TERMFLAG, "DEDENT"), ""), 1)
# Python lexical dictionary.
### MUST HANDLE WHOLELY BLANK LINES CORRECTLY!
def RMATCH(re, key, start=0):
group = re.match(key, start)
if group is None: return -1
return group.end() - group.start()
class pylexdict(kjParser.LexDictionary):
def __init__(self):
kjParser.LexDictionary.__init__(self)
# need to add special map for >>
self.brackets = 0 # count of active brackets
self.realindex = 0 # where to start
self.indents = [""] # stack of indents (start with a fake one)
self.lineno = 0
self.atdedent = 0
### handle multiple dedents correctly!!!
### translate tabs to 8 spaces...
from kjParser import TERMFLAG
self.NAMEflag = (TERMFLAG, "NAME")
self.STRINGflag = (TERMFLAG, "STRING")
self.NEWLINEflag = (TERMFLAG, "NEWLINE")
self.INDENTflag = (TERMFLAG, "INDENT")
self.DEDENTflag = (TERMFLAG, "DEDENT")
self.NUMBERflag = (TERMFLAG, "NUMBER")
def endoffile(self, String):
# pop off all indentations!
indents = self.indents
#lastresult = self.lastresult
self.realindex = len(String)
if not indents:
# pop indents
#print "eof after dedent"
result = self.lastresult = (ENDOFFILETERM, 0)
else:
#print "eof as dedent after", self.lastresult
del indents[-1]
if indents:
dedent = indents[-1]
else:
dedent = ""
result = self.lastresult = ((self.DEDENTflag, dedent), 1)
#print "returning eof", result, "after", lastresult
return result
def Token(self, String, StartPosition):
#print "Token", (StartPosition,
# `String[self.realindex:self.realindex+20]`, self.lastresult)
# HAVE TO FAKE OUT LEXER FOR DEDENTS
# STARTPOSITION COUNTS # OF TOKEN, NOT STRING POSITION
# STRING POSITION IS MAINTAINED IN LexDict object.
lastindex = self.lastindex
lastresult = self.lastresult
if self.laststring is not String:
#print "parsing new string"
self.laststring = String
# special hack: skip lead whitespace
cursor = 0
self.lineno = 1
while 1:
test = RMATCH(pywhitein,String, cursor)
if test<0: break
next = cursor + test
#print "lead skip:", next, String[cursor:next]
if String[next]!="\n": break
#skipped = String[cursor:next]
#if "\n" in skipped:
# self.lineno = (
# self.lineno + len(string.splitfields(skipped, "\n")))
#self.lineno = self.lineno+1
cursor = next + 1
self.realindex = cursor
self.saveindex = 0
self.indents = [""] # stack of indents (start with a fake one)
# pretend we saw a newline
self.lastresult = newlineresult
if StartPosition!=0:
self.laststring = None
raise ValueError, "python lexical parsing must start at zero"
lastindex = self.lastindex
lastresult = None
elif lastindex == StartPosition:
#print "returning lastresult ", lastresult
return lastresult
elif lastindex != StartPosition-1:
raise ValueError, "python lexer can't skip tokens"
#print "parsing", StartPosition, lastresult
# do newline counting here!
delta = String[self.saveindex: self.realindex]
#print "delta", `delta`
if "\n" in delta:
#print self.lineno, self.saveindex, self.realindex, `delta`
self.lineno = self.lineno + len(
string.splitfields(delta, "\n")) - 1
realindex = self.saveindex = self.realindex
self.lastindex = StartPosition
# skip whitespace (including comments)
### needs to be improved to parse blank lines, count line numbers...
# skip all totally blank lines (don't eat last newline)
atlineend = (String[realindex:realindex+1] == "\n"
or lastresult is newlineresult
or self.atdedent)
skipnewlines = (lastresult is newlineresult or
self.atdedent or
self.brackets>0)
if atlineend: #String[realindex:realindex+1]=="\n":
#print "trying to skip blank lines", String[realindex:realindex+10]
while 1:
#if String[realindex:realindex+1]=="\n":
# start = realindex+1 # move past current newline
# self.lineno = self.lineno + 1
#else:
# start = realindex
start = realindex
if skipnewlines:
while String[start:start+1]=="\n":
start = start+1
#self.lineno = self.lineno+1
#print "matching", `String[start:start+10]`
skip = RMATCH(pywhitein,String, start)
#print "skip=", skip
if skip<0: break
rs = skip + realindex + (start-realindex)
if rs==realindex: break
#print "at", rs, `String[rs]`
if (rs<len(String) and
(String[rs] == "\n" or
(skipnewlines and String[rs-1:rs]=="\n"))):
#print "skipping blank line"
#if lastresult is newlineresult or self.brackets>0:
# rs = rs + 1
#skipped = String[start:rs]
#if "\n" in skipped:
#self.lineno = self.lineno + len(
# string.splitfields(skipped, "\n"))
self.realindex = realindex = rs
#self.lineno = self.lineno+1
else:
if skipnewlines: self.realindex = realindex = start
break
#print "after skipping blank lines", `String[realindex:realindex+20]`
skipto = realindex
skip = 0
if self.brackets>0:
while 1:
#print "skipping white in brackets", skipto
if realindex>len(String):
break
if String[skipto]=="\n":
#self.lineno = self.lineno+1
skipto = skipto + 1
self.realindex = realindex = skipto
continue
skip = RMATCH(pywhiteout,String, skipto)
nextskipto = skipto+skip
#skipped = String[skipto:nextskipto]
#if "\n" in skipped:
# self.lineno = self.lineno+len(
# string.splitfields(skipped, "\n"))
if skip>0:
skipto = nextskipto
else: break
skip = skipto - realindex
elif not atlineend:
skip = RMATCH(pywhitein,String, realindex)
if skip<=0:
skip = 0
else:
#print "skipping", skip
nextri = realindex + skip
#skipped = String[realindex:nextri]
#if "\n" in skipped:
# self.lineno = self.lineno + len(
# string.splitfields(skipped, "\n"))
realindex = self.realindex = nextri
if realindex>=len(String):
return self.endoffile(String)
# now look for a keyword, name, number, punctuation,
# INDENT, DEDENT, NEWLINE
first = String[realindex]
#if last parse was newline and not in brackets:
# look for indent/dedent
if (self.brackets<=0 and (lastresult is newlineresult or self.atdedent)
and first != "\n"):
#print "looking for dent", realindex, `String[realindex:realindex+20]`
match = RMATCH(indentp,String, realindex)
if match>=0:
dent = String[realindex: realindex+match]
#print "dent match", match, `dent`
oldindex = realindex
self.realindex = realindex = realindex+match
# replace tabs with 8 spaces
dent = string.joinfields(string.splitfields(dent, "\t"),
" ")
dents = self.indents
lastdent = dents[-1]
ldl = len(lastdent)
dl = len(dent)
#print "last", ldl, dents
if ldl<dl:
self.atdedent = 0
result = self.lastresult = ((self.INDENTflag, dent), 1)
dents.append(dent)
#print "indent ", result, dents
return result
if ldl>dl:
self.realindex = oldindex # back up, may have to see it again!
self.atdedent = 1
result = self.lastresult = ((self.DEDENTflag, dent), 1)
del dents[-1]
#print "dedent ", result, dl, dents
return result
# otherwise, indentation is same, keep looking
# might be at eof now:
if realindex>=len(String):
#print "returning eof"
return self.endoffile(String)
first = String[realindex]
self.atdedent = 0
from string import digits#, letters
if(firstinpunctlistand
# special case for .123 numbers (yuck!)
(first!="." or String[realindex+1] not in digits)):
# is it a 2 char punct?
first2 = String[realindex:realindex+2]
if first2 in punct2:
result = self.lastresult = kwmap[first2]
self.realindex = realindex+2
#print "2 digit punct", result
return result
# otherwise, just return normal punct
result = self.lastresult = kwmap[first]
self.realindex = self.realindex + 1
### special bookkeeping
if first=="\n":
result = newlineresult
#print "newline!"
#self.lineno = self.lineno+1
elif first in "[{(":
#print "bracket!"
self.brackets = self.brackets + 1
elif first in "]})":
#print "close bracket!"
self.brackets = self.brackets - 1
#print "1 digit punct", result
return result
if first in digits or first==".":
# parse a number...
skip = numterm.search(String, realindex)
if skip<=realindex:
raise parseerror, "number length<1 (!)"
thenumber = String[realindex:skip]
self.realindex = skip
### note don't interpret number here!!
result = self.lastresult = ((self.NUMBERflag, thenumber), 1)
#print "number", result
return result
if first in alpha:
# try keyword...
first2 = String[realindex: realindex+2]
if first2 in char2kw:
if String[realindex+2:realindex+3] not in id_letters:
# parse a 2 char kw first2
result = self.lastresult = kwmap[first2]
self.realindex = self.realindex+2
#print "keyword 2", result
return result
first3 = String[realindex: realindex+3]
if char3kwdict.has_key(first3):
the_kw = char3kwdict[first3]
the_end = realindex+len(the_kw)
if ((the_end<len(String)) and
(String[the_end] not in id_letters) and
(String[realindex:the_end]==the_kw)):
# parse the_kw
self.realindex = the_end
result = self.lastresult = kwmap[the_kw]
#print "keyword +", result
return result
#otherwise parse an identifier
#print "looking for name:", `String[realindex:realindex+10]`
skip = nameterm.search(String, realindex)
if skip<=realindex:
raise parseerror, "identifier length<1 (!)"
theid = String[realindex:skip]
self.realindex = skip
### note don't interpret number here!!
result = self.lastresult = ((self.NAMEflag, theid), 1)
#print "id", result
return result
if first in "\"'":
# check for triplequotes
first3 = first*3
if String[realindex: realindex+3] == first3:
# parse triple quotes
start = place = realindex+3
while 1:
last = string.find(String, first3, place)
if last<0:
raise parseerror, "failed to terminate triple quotes"
if String[last-1:last]=="\\" and String[last-2:last-1]!="\\":
place = last+1
else: break
the_string = String[start: last]
self.realindex = last+3
result = self.lastresult = ((self.STRINGflag, the_string), 1)
#print "3q string", result
# count the newlines!
#newlinecount = len(string.splitfields(the_string, "\n"))
#self.lineno = self.lineno+newlinecount
#print "triple quotes", result
return result
else:
# parse single quotes
sanity = start = place = realindex+1
done = 0
while 1:
sanity = min(string.find(String, "\n", sanity), len(String))
if sanity<start:
sanity=len(String)
break
if String[sanity-1]!="\\":
break
else:
#self.lineno = self.lineno+1
sanity = sanity + 1
while 1:
last = string.find(String, first, place)
if last<0 or last>sanity:
raise parseerror, "failed to terminate single quotes"
if String[last-1:last]=="\\":
# are we at the end of an odd number of backslashes? (yuck!)
bplace = last-1
while String[bplace:bplace+1]=="\\":
bplace = bplace-1
if (last-bplace)%2==1:
break # the end quote is real!
place = last+1
else: break
the_string = String[start:last]
self.realindex = last+1
result = self.lastresult = ((self.STRINGflag, the_string), 1)
#print "1q string", result
return result
#print (String[realindex-20:realindex-1], String[realindex],
# String[realindex+1:realindex+20])
raise parseerror, "invalid first: " + `first`
# use a modified lexstringwalker
class pylexstringwalker(kjParser.LexStringWalker):
def DUMP(self):
kjParser.DumpStringWindow(self.String, self.LexDict.realindex)
## a HORRIBLE HACK! of a hack: override the DoParse of Grammar
## to give Python line numbers. RELIES ON GLOBAL pyg
##
def hackDoParse(String, Context=None, DoReductions=1):
import sys, kjParser
try:
# construct the ParserObj
# add a newline to front to avoid problem with leading comment
#String = "\n%s\n" % String
Stream = pylexstringwalker( String, pyg.LexD )
Stack = [] # {-1:0} #Walkers.SimpleStack()
ParseOb = kjParser.ParserObj( pyg.RuleL, Stream, pyg.DFA, Stack, \
DoReductions, Context )
# do the parse
ParseResult = ParseOb.GO()
# return final result of reduction and the context
return (ParseResult[1], Context)
#return kjParser.Grammar.DoParse(pyg, String, Context, DoReductions)
except: ### for testing!!
t, v = sys.exc_type, sys.exc_value
v = ("near line", pyg.LexD.lineno, v)
raise t, v
buildinfo = """
Please edit the ARCHIVE parameter of this module (%s)
to place the python grammar archive in a standard
directory to prevent the module from rebuilding
the python grammar over and over and over...
""" % __name__
def GrammarBuild():
global pyg
import kjParseBuild
pyg = kjParseBuild.NullCGrammar()
pyg.DoParse = hackDoParse
# override lexical dict here
pyg.LexD = pylexdict()
DeclareTerminals(pyg)
pyg.Keywords(keywords)
pyg.punct("~!#%^&*()-+=|{}'`<>,.;:/[]{}")
pyg.Nonterms(nonterms)
pyg.Declarerules(pyrules)
print buildinfo
print "compiling... this may take a while..."
pyg.Compile()
print "dumping"
outfile = open(marshalfilename, "wb")
pyg.MarshalDump(outfile)
outfile.close()
print "self testing the grammar"
test(pyg)
print "\n\ndone with regeneration"
return pyg
def unMarshalpygram():
global pyg
import kjParser
print "loading"
try:
infile = open(marshalfilename, "rb")
except IOError:
print marshalfilename, "not found, attempting creation"
pyg = GrammarBuild()
else:
pyg = kjParser.UnMarshalGram(infile)
infile.close()
pyg.DoParse = hackDoParse
# lexical override
pyg.LexD = pylexdict()
DeclareTerminals(pyg)
# BindRules(pyg)
if dotest:
print "self testing the grammar"
test(pyg)
return pyg
# not used, commented
#### interpretation rules/classes
#
#def zeroth(list, Context):
# return list[0] # eg, for all1, ignore all but first
#
## file_input, stmt, simple_stmt, compound_stmt give list of statement_ob
#def append(list, Context):
# "eg, for top_stmt, conjoin two smt lists"
# return list[0] + list[1]
#
## file_input >zeroth
#
## simple, compound, one_small, small_semi: echol
#def echol(list, Context):
# return list
#
## more_small > seq_sep
#def seq_sep(list, Context):
# list[0].append(list[2])
# return list[0]
#
## smexpr, smassn, smpring, smdel, smflow, smimport, smglobal, smexec
## > zeroth
#
## cmif, cmwhile, cmfor, cmtry, cmdef, cmclass > zeroth
#
#
#def BindRules(pyg):
# for name in string.split("""
# all1 file_input cmif cmwhile cmfor cmtry cmdef cmclass
# smexpr smassn smprint smdel smflow smimport smglobal smexec
# """):
# pyg.Bind(name, zeroth)
# for name in string.split("""
# simple compound one_small small_semi
# """):
# pyg.Bind(name, echol)
# pyg.Bind("top_stmt", append)
# pyg.Bind("more_small", seq_sep)
teststring = """#
#
# a test string
#
from string import join, split
'''
import re
for a in l:
a.attr, a[x], b = c
else:
d = b
'''
class zzz:
'''
#doc string
'''
'''
global re, join
d = {}
for i in range(10): d[i] = i
'''
def test(c,s):
return "this"
while not done:
print done
break
list = [1,2,3]
# comment
return 5
n,x = 89 >> 90 + 6 / 7 % x + z << 6 + 2 ** 8
if x==5:
while y:
for i in range(6):
raise SystemError, "oops"
"""
#teststring ="""\
## comment
#if x in y: print z
#elif 1: print w
#"""
'''
teststring="""
exec "print 1"
"""
'''
def test(grammar, context=None, teststring=teststring):
from time import time
now = time()
x = grammar.DoParse1(teststring, context)
elapsed = time()-now
print x
print elapsed
return x
regen = 0
dotest = 0
if __name__ == "__main__" :
if regen: GrammarBuild()
unMarshalpygram()
|