Initial revision
This commit is contained in:
Hans Lambermont
432
intern/python/modules/simpleparse/generator.py
Normal file
432
intern/python/modules/simpleparse/generator.py
Normal file
@@ -0,0 +1,432 @@
|
||||
from TextTools.TextTools import *
|
||||
import bootstrap # the hand-coded parser
|
||||
import operator, strop as string
|
||||
|
||||
def err( value ):
|
||||
print value
|
||||
|
||||
class _BaseGenerator:
|
||||
'''
|
||||
Class providing the functions required to turn a
|
||||
parse tree as generated by the bootstrap parser into
|
||||
a new set of parser tuples. I.e a parser generator :)
|
||||
Effectively this is the bootstrap generator.
|
||||
'''
|
||||
def __init__( self, syntaxstring = bootstrap.declaration, parserelement = 'declarationset' ):
|
||||
'''
|
||||
Turn syntaxstring into a parsetree using
|
||||
the bootstrap module's parse command
|
||||
'''
|
||||
# should do some error checking in here :)
|
||||
self.syntaxstring = syntaxstring
|
||||
self.parsetree = bootstrap.parse( syntaxstring, parserelement )[1][0] # the child list
|
||||
self.nameset = []
|
||||
self.tupleset = []
|
||||
def stringval( self, tuple ):
|
||||
'''
|
||||
Return the string value for a parse-result tuple
|
||||
'''
|
||||
return self.syntaxstring[ tuple[1]:tuple[2] ]
|
||||
def build( self, prebuiltnodes=() ):
|
||||
'''
|
||||
Build a new parsing table from the syntax string.
|
||||
New parsers may be accessed using the parserbyname method.
|
||||
|
||||
The pre-built nodes are parsing tables for inclusion in the grammar
|
||||
Added version 1.0.1 to provide greater extensibility.
|
||||
'''
|
||||
# first register all declared names to reserve their indicies
|
||||
#if self.__class__.__name__ == 'Generator':
|
||||
# import pdb
|
||||
# pdb.set_trace()
|
||||
for key, value in prebuiltnodes:
|
||||
self.nameset.append( key )
|
||||
self.tupleset.append( value )
|
||||
for decl in self.parsetree[3]:
|
||||
#print decl
|
||||
name = self.stringval( decl[3][0] )
|
||||
self.nameset.append( name )
|
||||
self.tupleset.append( None)
|
||||
#print 'Declared names:',self.nameset
|
||||
for i in range( len( self.nameset)):
|
||||
#print '''Processing declaration %s '''% self.nameset[i]
|
||||
dataset = self.group( ('group',1,2, self.parsetree[3][i][3][1:]), self )
|
||||
if dataset:
|
||||
self.tupleset[i] = tuple( dataset)
|
||||
def parserbyname( self, name ):
|
||||
'''
|
||||
Retrieve a single parsing tuple by its production name
|
||||
'''
|
||||
try:
|
||||
return self.tupleset[ self.nameset.index( name ) ]
|
||||
except ValueError:
|
||||
print '''Could not find parser tuple of name''', name
|
||||
return ()
|
||||
def allparsers (self):
|
||||
'''
|
||||
Return a list of (productionname, parsingtuple) values
|
||||
suitable for passing to another generator as its pre-calculated
|
||||
set of parsing tuples. (See method build)
|
||||
'''
|
||||
returnvalue = []
|
||||
for i in range(len( self.nameset)):
|
||||
returnvalue.append ( (self.nameset[i],self.tupleset[i]) )
|
||||
return returnvalue
|
||||
### Actual processing functions...
|
||||
def element_token( self, eltup, genobj, reportname=None ):
|
||||
# Determine the type of element
|
||||
# Descry the various options for the element
|
||||
negative = optional = repeating = element = None
|
||||
for data in eltup[3]:
|
||||
if data[0] == 'negpos_indicator':
|
||||
if genobj.stringval ( data ) == '-':
|
||||
negative = 1
|
||||
elif data[0] == 'occurence_indicator':
|
||||
data = genobj.stringval ( data )
|
||||
if data == '*':
|
||||
optional = 1
|
||||
repeating = 1
|
||||
elif data == '+':
|
||||
repeating = 1
|
||||
elif data == '?':
|
||||
optional = 1
|
||||
else:
|
||||
err( 'Unknown occurence indicator '+ data )
|
||||
else:
|
||||
element = data
|
||||
# call the appropriate handler
|
||||
try:
|
||||
return getattr( self, element [0])( element, genobj, negative, repeating, optional)
|
||||
except AttributeError,x:
|
||||
err( '''Didn't find handler for element type %s, parser build aborted'''%element [0])
|
||||
raise x
|
||||
|
||||
def group( self, els, genobj, negative= None, repeating=None, optional = None, reportname=None):
|
||||
'''
|
||||
Determine what type of group we're dealing with and determine what
|
||||
function to call, then call it.
|
||||
'''
|
||||
groupset = els[3]
|
||||
# groupset is an element_token followed by a possible added_token
|
||||
if groupset:
|
||||
els = []
|
||||
els.append( groupset[0] )
|
||||
if len(groupset) > 1:
|
||||
els[len(els):] = groupset[1][3]
|
||||
gtype = groupset[1][0]
|
||||
if gtype == 'seq_added_token':
|
||||
return self.seq( els, genobj, negative, repeating, optional, reportname )
|
||||
elif gtype == 'fo_added_token':
|
||||
return self.fo( els, genobj, negative, repeating, optional, reportname )
|
||||
else:
|
||||
err( '''An as-yet undefined group type was used! %s'''%gtype )
|
||||
else: # default "sequence" of one... could do more work and make it process the results specifically, but that's optimisation ;)
|
||||
return self.seq( els, genobj, negative, repeating, optional, None )
|
||||
else:
|
||||
return []
|
||||
|
||||
|
||||
def seq( self, els, genobj, negative= None, repeating=None, optional = None, reportname=None ):
|
||||
elset = map( self.element_token, els, [genobj]*len( els) )
|
||||
elset = reduce( operator.add, elset )
|
||||
if negative:
|
||||
if repeating:
|
||||
if optional:
|
||||
return [(None, SubTable, (( None, SubTable,( (None, SubTable, tuple( elset), 2,1), (None, Fail, Here),(None,Skip,1) ), 2,1 ), ( None, EOF, Here, -1,1 ), ), ), ]
|
||||
else: # not optional
|
||||
return [(None, SubTable, (( None, SubTable,( (None, SubTable, tuple( elset), 2,1), (None, Fail, Here),(None,Skip,1) )), ( None, SubTable,( (None, SubTable, tuple( elset), 2,1), (None, Fail, Here),(None,Skip,1) ), 2,1 ), ( None, EOF, Here, -1,1 ), ), ), ]
|
||||
else: # single
|
||||
if optional:
|
||||
return [ (None, SubTable, ( (None, SubTable, tuple( elset), 2,1), (None, Fail, Here), (None, Skip, 1) ),1,1) ]
|
||||
else: # not optional
|
||||
return [ (None, SubTable, ( (None, SubTable, tuple( elset), 2,1), (None, Fail, Here), (None, Skip, 1) )) ]
|
||||
else: # positive
|
||||
if repeating:
|
||||
if optional:
|
||||
return [ (None, SubTable, tuple( elset), 1,0) ]
|
||||
else: # not optional
|
||||
|
||||
return [ (None, SubTable, tuple( elset)), (None, SubTable, tuple( elset), 1,0) ]
|
||||
else: # single
|
||||
if optional:
|
||||
return [ (None, SubTable, tuple( elset), 1,1) ]
|
||||
else: # not optional
|
||||
return [ (None, SubTable, tuple( elset)) ]
|
||||
|
||||
def fo( self, els, genobj, negative= None, repeating=None, optional = None, reportname=None ):
|
||||
elset = map( self.element_token, els, [genobj]*len( els) )
|
||||
elset = reduce( operator.add, elset )
|
||||
elset = []
|
||||
for el in els:
|
||||
dataset = self.element_token( el, genobj )
|
||||
if len( dataset) == 1 and len(dataset[0]) == 3: # we can alter the jump states with impunity
|
||||
elset.append( dataset[0] )
|
||||
else: # for now I'm eating the inefficiency and doing an extra SubTable for all elements to allow for easy calculation of jumps within the FO group
|
||||
elset.append( (None, SubTable, tuple( dataset )) )
|
||||
if negative:
|
||||
# all negative FO's have the meaning "a positive, single, non-optional FO not matching"
|
||||
# the flags modify how failure and continuation are handled in that case, so they can use
|
||||
# the same procset.
|
||||
# Note: Negative FO groups are _very_ heavy, they have normally about 4 subtable calls
|
||||
# guess we'll find out how well mxTextTools handles recursive tables :)
|
||||
procset = []
|
||||
for i in range( len( elset) -1): # note that we have to treat last el specially
|
||||
ival = elset[i] + (1,len(elset)-i)
|
||||
procset.append( ival ) # if success, jump past end
|
||||
procset.append( elset[-1] + (2,1) ) # will cause a failure if last element doesn't match
|
||||
procset.append( (None, Fail, Here ) )
|
||||
procset.append( (None, Skip, 1) )
|
||||
# if the following looks familiar you probably looked at seq above
|
||||
if repeating:
|
||||
if optional:
|
||||
return [ (None, SubTable, ( (None, SubTable, tuple( procset), 2,1), (None, EOF, Here,-1,1) ) ) ]
|
||||
else: # not optional
|
||||
return [ (None, SubTable, ( (None, SubTable, tuple( procset)),(None, SubTable, tuple( procset), 2,1), (None, EOF, Here,-1,1) ) ) ]
|
||||
else: # single
|
||||
if optional:
|
||||
return [ (None, SubTable, tuple( procset), 1,1) ]
|
||||
else: # not optional
|
||||
return [ (None, SubTable, tuple( procset) ) ]
|
||||
else: # positive
|
||||
if repeating:
|
||||
if optional:
|
||||
procset = []
|
||||
for i in range( len( elset)):
|
||||
procset.append( elset[i] + (1,-i) ) # if success, go back to start which is -i elements back
|
||||
return procset
|
||||
else: # not optional
|
||||
procset = []
|
||||
for i in range( len( elset)-1):
|
||||
procset.append( elset[i] + (1, len(elset)-i+1) ) # if success, jump to later section
|
||||
procset.append( elset[-1] + ( 1, 2) ) # will cause a failure if last element doesn't match using an explicit fail command
|
||||
procset.append( (None, Fail, Here) ) # will cause a failure if last element doesn't match using an explicit fail command
|
||||
for i in range( len( elset)-1):
|
||||
procset.append( elset[i] + (1, -i) ) # if success, go back to start which is -i elements back
|
||||
procset.append( elset[-1] + ( 1, 1-(len(elset)) ) ) # will cause a failure if last element doesn't match using an explicit fail command
|
||||
return procset
|
||||
else: # single
|
||||
if optional:
|
||||
procset = []
|
||||
for i in range( len( elset)):
|
||||
procset.append( elset[i] + (1,len(elset)-i) ) # if success, jump past end
|
||||
return procset
|
||||
else: # not optional
|
||||
procset = []
|
||||
for i in range( len( elset) -1): # note that we have to treat last el specially
|
||||
procset.append( elset[i] + (1,len(elset)-i) ) # if success, jump past end
|
||||
procset.append( elset[-1] ) # will cause a failure if last element doesn't match
|
||||
return procset
|
||||
|
||||
def name( self, value, genobj, negative = None, repeating = None, optional = None, reportname=None ):
|
||||
svalue = genobj.stringval( value )
|
||||
try:
|
||||
sindex = genobj.nameset.index( svalue )
|
||||
except ValueError: # eeps, a value not declared
|
||||
try:
|
||||
sindex = genobj.nameset.index( '<'+svalue+'>' )
|
||||
svalue = None
|
||||
except ValueError:
|
||||
err( '''The name %s could not be found in the declarationset. The parser will not compile.'''%svalue)
|
||||
genobj.nameset.append( svalue )
|
||||
genobj.tupleset.append( None )
|
||||
sindex = len( genobj.nameset) - 1
|
||||
if negative:
|
||||
if repeating:
|
||||
if optional:
|
||||
return [ (svalue, SubTable, ( (None, TableInList, (genobj.tupleset, sindex), 1,3), (None, EOF, Here,1,2), (None,Skip,1,-2,-2) ) ) ]
|
||||
else: # not optional
|
||||
return [ (svalue, SubTable, ( (None, TableInList, (genobj.tupleset, sindex),2,1),(None, Fail, Here),(None, Skip, 1), (None, TableInList, (genobj.tupleset, sindex), 1,3), (None, EOF, Here,1,2), (None,Skip,1,-2,-2) ) ) ]
|
||||
else: # single
|
||||
if optional:
|
||||
return [ (None, SubTable, ( (None, TableInList, (genobj.tupleset, sindex),2,1),(None, Fail, Here),(svalue, Skip, 1) ),1,1) ]
|
||||
else: # not optional
|
||||
return [ (None, SubTable, ( (None, TableInList, (genobj.tupleset, sindex),2,1),(None, Fail, Here),(svalue, Skip, 1) )) ]
|
||||
else: # positive
|
||||
if repeating:
|
||||
if optional:
|
||||
return [ (svalue, TableInList, (genobj.tupleset, sindex), 1,0) ]
|
||||
else: # not optional
|
||||
return [ (svalue, TableInList, (genobj.tupleset, sindex)), (svalue, TableInList, (genobj.tupleset, sindex),1,0) ]
|
||||
else: # single
|
||||
if optional:
|
||||
return [ (svalue, TableInList, (genobj.tupleset, sindex), 1,1) ]
|
||||
else: # not optional
|
||||
return [ (svalue, TableInList, (genobj.tupleset, sindex)) ]
|
||||
specialescapedmap = {
|
||||
'a':'\a',
|
||||
'b':'\b',
|
||||
'f':'\f',
|
||||
'n':'\n',
|
||||
'r':'\r',
|
||||
't':'\t',
|
||||
'v':'\v',
|
||||
'\\':'\\',
|
||||
'"':'"',
|
||||
"'":"'",
|
||||
}
|
||||
|
||||
def escapedchar( self, el, genobj ):
|
||||
svalue = ''
|
||||
if el[3][0][0] == 'SPECIALESCAPEDCHAR':
|
||||
svalue = svalue + self.specialescapedmap[ genobj.stringval( el[3][0] ) ]
|
||||
elif el[3][0][0] == 'OCTALESCAPEDCHAR':
|
||||
#print 'OCTALESCAPEDCHAR', genobj.stringval( el)
|
||||
ovnum = 0
|
||||
ovpow = 0
|
||||
ov = genobj.stringval( el[3][0] )
|
||||
while ov:
|
||||
ovnum = ovnum + int( ov[-1] ) * (8**ovpow)
|
||||
ovpow = ovpow + 1
|
||||
ov = ov[:-1]
|
||||
svalue = svalue + chr( ovnum )
|
||||
#print 'svalue ', `svalue`
|
||||
return svalue
|
||||
|
||||
|
||||
def literal( self, value, genobj, negative = None, repeating=None, optional=None, reportname=None ):
|
||||
'''
|
||||
Calculate the tag-table for a literal element token
|
||||
'''
|
||||
svalue = ''
|
||||
for el in value[3]:
|
||||
if el[0] in ('CHARNOSNGLQUOTE', 'CHARNODBLQUOTE'):
|
||||
svalue = svalue+genobj.stringval( el )
|
||||
elif el[0] == 'ESCAPEDCHAR':
|
||||
svalue = svalue + self.escapedchar( el, genobj )
|
||||
#print 'literal value', `genobj.stringval( value )`
|
||||
#print ' svalue', `svalue`
|
||||
# svalue = svalue[1:-1]
|
||||
if negative:
|
||||
if repeating: # a repeating negative value, a "search" in effect
|
||||
if optional: # if fails, then go to end of file
|
||||
return [ (None, sWordStart, BMS( svalue ),1,2), (None, Move, ToEOF ) ]
|
||||
else: # must first check to make sure the current position is not the word, then the same
|
||||
return [ (None, Word, svalue, 2,1),(None, Fail, Here),(None, sWordStart, BMS( svalue ),1,2), (None, Move, ToEOF ) ]
|
||||
#return [ (None, Word, svalue, 2,1),(None, Fail, Here),(None, WordStart, svalue,1,2), (None, Move, ToEOF ) ]
|
||||
else: # a single-character test saying "not a this"
|
||||
if optional: # test for a success, move back if success, move one forward if failure
|
||||
if len(svalue) > 1:
|
||||
return [ (None, Word, svalue, 2,1),
|
||||
(None, Skip, -len(svalue), 2,2), # backup if this was the word to start of word, succeed
|
||||
(None, Skip, 1 ) ] # else just move one character and succeed
|
||||
else: # Uses Is test instead of Word test, should be faster I'd imagine
|
||||
return [ (None, Is, svalue, 2,1),
|
||||
(None, Skip, -1, 2,2), # backtrack
|
||||
(None, Skip, 1 ) ] # else just move one character and succeed
|
||||
else: # must find at least one character not part of the word, so
|
||||
if len(svalue) > 1:
|
||||
return [ (None, Word, svalue, 2,1),
|
||||
(None, Fail, Here),
|
||||
(None, Skip, 1 ) ] # else just move one character and succeed
|
||||
else: #must fail if it finds or move one forward
|
||||
return [ (None, Is, svalue, 2,1),
|
||||
(None, Fail, Here),
|
||||
(None, Skip, 1 ) ] # else just move one character and succeed
|
||||
else: # positive
|
||||
if repeating:
|
||||
if optional:
|
||||
if len(svalue) > 1:
|
||||
return [ (None, Word, svalue, 1,0) ]
|
||||
else:
|
||||
return [ (None, Is, svalue, 1,0) ]
|
||||
else: # not optional
|
||||
if len(svalue) > 1:
|
||||
return [ (None, Word, svalue),(None, Word, svalue,1,0) ]
|
||||
else:
|
||||
return [ (None, Is, svalue),(None, Is, svalue,1,0) ]
|
||||
else: # not repeating
|
||||
if optional:
|
||||
if len(svalue) > 1:
|
||||
return [ (None, Word, svalue, 1,1) ]
|
||||
else:
|
||||
return [ (None, Is, svalue, 1,1) ]
|
||||
else: # not optional
|
||||
if len(svalue) > 1:
|
||||
return [ (None, Word, svalue) ]
|
||||
else:
|
||||
return [ (None, Word, svalue) ]
|
||||
|
||||
def charnobrace( self, cval, genobj ):
|
||||
#print 'cval', cval
|
||||
if cval[3][0][0] == 'ESCAPEDCHAR':
|
||||
return self.escapedchar( cval[3][0], genobj )
|
||||
#print '''Straight non-brace character''', `genobj.stringval( cval[3][0] )`
|
||||
return genobj.stringval( cval )
|
||||
def range( self, value, genobj, negative = None, repeating=None, optional=None, reportname=None ):
|
||||
dataset = []
|
||||
for cval in value[3]:
|
||||
if cval[0] == 'CHARBRACE':
|
||||
dataset.append( ']')
|
||||
elif cval[0] == 'CHARDASH':
|
||||
dataset.append( '-')
|
||||
elif cval[0] == 'CHARNOBRACE':
|
||||
dataset.append( self.charnobrace( cval, genobj ) )
|
||||
elif cval[0] == 'CHARRANGE':
|
||||
start = ord( self.charnobrace( cval[3][0], genobj ) )
|
||||
end = ord( self.charnobrace( cval[3][1], genobj ) )
|
||||
if start < end:
|
||||
dataset.append( string.join( map( chr, range( start, end +1 ) ), '' ) )
|
||||
else:
|
||||
dataset.append( string.join( map( chr, range( end, start +1 ) ), '' ) )
|
||||
else:
|
||||
dataset.append( genobj.stringval( cval ) )
|
||||
if negative:
|
||||
#svalue = set( string.join( dataset, '' ), 0 )
|
||||
svalue = string.join( dataset, '' )
|
||||
else:
|
||||
#svalue = set( string.join( dataset, '' ), 1)
|
||||
svalue = string.join( dataset, '' )
|
||||
if negative:
|
||||
if repeating:
|
||||
if optional:
|
||||
#return [ (None, AllInSet, svalue, 1 ) ]
|
||||
return [ (None, AllNotIn, svalue, 1 ) ]
|
||||
else: # not optional
|
||||
#return [ (None, AllInSet, svalue ) ]
|
||||
return [ (None, AllNotIn, svalue ) ]
|
||||
else: # not repeating
|
||||
if optional:
|
||||
#return [ (None, IsInSet, svalue, 1 ) ]
|
||||
return [ (None, IsNotIn, svalue, 1 ) ]
|
||||
else: # not optional
|
||||
#return [ (None, IsInSet, svalue ) ]
|
||||
return [ (None, IsNotIn, svalue ) ]
|
||||
else:
|
||||
if repeating:
|
||||
if optional:
|
||||
#return [ (None, AllInSet, svalue, 1 ) ]
|
||||
return [ (None, AllIn, svalue, 1 ) ]
|
||||
else: # not optional
|
||||
#return [ (None, AllInSet, svalue ) ]
|
||||
return [ (None, AllIn, svalue ) ]
|
||||
else: # not repeating
|
||||
if optional:
|
||||
#return [ (None, IsInSet, svalue, 1 ) ]
|
||||
return [ (None, IsIn, svalue, 1 ) ]
|
||||
else: # not optional
|
||||
#return [ (None, IsInSet, svalue ) ]
|
||||
return [ (None, IsIn, svalue ) ]
|
||||
|
||||
class Generator( _BaseGenerator ):
|
||||
def __init__( self, syntaxstring , parser ):
|
||||
self.syntaxstring = syntaxstring
|
||||
self.parsetree = [0,1,2, tag( syntaxstring, parser )[1] ]
|
||||
self.nameset = []
|
||||
self.tupleset = []
|
||||
|
||||
def buildParser( declaration, prebuiltnodes=() ):
|
||||
'''
|
||||
End-developer function to create an application-specific parser
|
||||
the parsing tuple is available on the returned object as
|
||||
object.parserbyname( 'declaredname' ), where declaredname is the
|
||||
name you defined in your language defintion file.
|
||||
|
||||
The declaration argument is the text of a language defintion file.
|
||||
'''
|
||||
proc = _BaseGenerator( )
|
||||
proc.build()
|
||||
newgen = Generator( declaration, proc.parserbyname( 'declarationset' ) )
|
||||
newgen.build( prebuiltnodes=prebuiltnodes )
|
||||
return newgen
|
||||
|
||||
|
||||
Reference in New Issue
Block a user