| #----------------------------------------------------------------------------- |
| # ply: lex.py |
| # |
| # Author: David M. Beazley (dave@dabeaz.com) |
| # |
| # Copyright (C) 2001-2006, David M. Beazley |
| # |
| # This library is free software; you can redistribute it and/or |
| # modify it under the terms of the GNU Lesser General Public |
| # License as published by the Free Software Foundation; either |
| # version 2.1 of the License, or (at your option) any later version. |
| # |
| # This library is distributed in the hope that it will be useful, |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| # Lesser General Public License for more details. |
| # |
| # You should have received a copy of the GNU Lesser General Public |
| # License along with this library; if not, write to the Free Software |
| # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
| # |
| # See the file COPYING for a complete copy of the LGPL. |
| #----------------------------------------------------------------------------- |
| |
| __version__ = "2.2" |
| |
| import re, sys, types |
| |
| from . import util |
| import collections |
| |
| |
| # Regular expression used to match valid token names |
| _is_identifier = re.compile(r'^[a-zA-Z0-9_]+$') |
| |
| # Available instance types. This is used when parsers are defined by a class. |
| # In Python3 the InstanceType and ObjectType are no more, they've passed, ceased |
| # to be, they are ex-classes along with old-style classes |
| |
| try: |
| _INSTANCETYPE = (types.InstanceType, types.ObjectType) |
| except AttributeError: |
| _INSTANCETYPE = object |
| |
| # Exception thrown when invalid token encountered and no default error |
| # handler is defined. |
| class LexError(Exception): |
| def __init__(self,message,s): |
| self.args = (message,) |
| self.text = s |
| |
| # Token class |
| class LexToken(object): |
| def __str__(self): |
| return "LexToken(%s,%r,%d,%d)" % (self.type,self.value,self.lineno,self.lexpos) |
| def __repr__(self): |
| return str(self) |
| def skip(self,n): |
| self.lexer.skip(n) |
| |
| # ----------------------------------------------------------------------------- |
| # Lexer class |
| # |
| # This class encapsulates all of the methods and data associated with a lexer. |
| # |
| # input() - Store a new string in the lexer |
| # token() - Get the next token |
| # ----------------------------------------------------------------------------- |
| |
| class Lexer: |
| def __init__(self): |
| self.lexre = None # Master regular expression. This is a list of |
| # tuples (re,findex) where re is a compiled |
| # regular expression and findex is a list |
| # mapping regex group numbers to rules |
| self.lexretext = None # Current regular expression strings |
| self.lexstatere = {} # Dictionary mapping lexer states to master regexs |
| self.lexstateretext = {} # Dictionary mapping lexer states to regex strings |
| self.lexstate = "INITIAL" # Current lexer state |
| self.lexstatestack = [] # Stack of lexer states |
| self.lexstateinfo = None # State information |
| self.lexstateignore = {} # Dictionary of ignored characters for each state |
| self.lexstateerrorf = {} # Dictionary of error functions for each state |
| self.lexreflags = 0 # Optional re compile flags |
| self.lexdata = None # Actual input data (as a string) |
| self.lexpos = 0 # Current position in input text |
| self.lexlen = 0 # Length of the input text |
| self.lexerrorf = None # Error rule (if any) |
| self.lextokens = None # List of valid tokens |
| self.lexignore = "" # Ignored characters |
| self.lexliterals = "" # Literal characters that can be passed through |
| self.lexmodule = None # Module |
| self.lineno = 1 # Current line number |
| self.lexdebug = 0 # Debugging mode |
| self.lexoptimize = 0 # Optimized mode |
| |
| def clone(self,object=None): |
| c = Lexer() |
| c.lexstatere = self.lexstatere |
| c.lexstateinfo = self.lexstateinfo |
| c.lexstateretext = self.lexstateretext |
| c.lexstate = self.lexstate |
| c.lexstatestack = self.lexstatestack |
| c.lexstateignore = self.lexstateignore |
| c.lexstateerrorf = self.lexstateerrorf |
| c.lexreflags = self.lexreflags |
| c.lexdata = self.lexdata |
| c.lexpos = self.lexpos |
| c.lexlen = self.lexlen |
| c.lextokens = self.lextokens |
| c.lexdebug = self.lexdebug |
| c.lineno = self.lineno |
| c.lexoptimize = self.lexoptimize |
| c.lexliterals = self.lexliterals |
| c.lexmodule = self.lexmodule |
| |
| # If the object parameter has been supplied, it means we are attaching the |
| # lexer to a new object. In this case, we have to rebind all methods in |
| # the lexstatere and lexstateerrorf tables. |
| |
| if object: |
| newtab = { } |
| for key, ritem in self.lexstatere.items(): |
| newre = [] |
| for cre, findex in ritem: |
| newfindex = [] |
| for f in findex: |
| if not f or not f[0]: |
| newfindex.append(f) |
| continue |
| newfindex.append((getattr(object,f[0].__name__),f[1])) |
| newre.append((cre,newfindex)) |
| newtab[key] = newre |
| c.lexstatere = newtab |
| c.lexstateerrorf = { } |
| for key, ef in self.lexstateerrorf.items(): |
| c.lexstateerrorf[key] = getattr(object,ef.__name__) |
| c.lexmodule = object |
| |
| # Set up other attributes |
| c.begin(c.lexstate) |
| return c |
| |
| # ------------------------------------------------------------ |
| # writetab() - Write lexer information to a table file |
| # ------------------------------------------------------------ |
| def writetab(self,tabfile): |
| tf = open(tabfile+".py","w") |
| tf.write("# %s.py. This file automatically created by PLY (version %s). Don't edit!\n" % (tabfile,__version__)) |
| tf.write("_lextokens = %s\n" % repr(self.lextokens)) |
| tf.write("_lexreflags = %s\n" % repr(self.lexreflags)) |
| tf.write("_lexliterals = %s\n" % repr(self.lexliterals)) |
| tf.write("_lexstateinfo = %s\n" % repr(self.lexstateinfo)) |
| |
| tabre = { } |
| for key, lre in self.lexstatere.items(): |
| titem = [] |
| for i in range(len(lre)): |
| titem.append((self.lexstateretext[key][i],_funcs_to_names(lre[i][1]))) |
| tabre[key] = titem |
| |
| tf.write("_lexstatere = %s\n" % repr(tabre)) |
| tf.write("_lexstateignore = %s\n" % repr(self.lexstateignore)) |
| |
| taberr = { } |
| for key, ef in self.lexstateerrorf.items(): |
| if ef: |
| taberr[key] = ef.__name__ |
| else: |
| taberr[key] = None |
| tf.write("_lexstateerrorf = %s\n" % repr(taberr)) |
| tf.close() |
| |
| # ------------------------------------------------------------ |
| # readtab() - Read lexer information from a tab file |
| # ------------------------------------------------------------ |
| def readtab(self,tabfile,fdict): |
| exec("import %s as lextab" % tabfile) |
| self.lextokens = lextab._lextokens |
| self.lexreflags = lextab._lexreflags |
| self.lexliterals = lextab._lexliterals |
| self.lexstateinfo = lextab._lexstateinfo |
| self.lexstateignore = lextab._lexstateignore |
| self.lexstatere = { } |
| self.lexstateretext = { } |
| for key,lre in lextab._lexstatere.items(): |
| titem = [] |
| txtitem = [] |
| for i in range(len(lre)): |
| titem.append((re.compile(lre[i][0],lextab._lexreflags),_names_to_funcs(lre[i][1],fdict))) |
| txtitem.append(lre[i][0]) |
| self.lexstatere[key] = titem |
| self.lexstateretext[key] = txtitem |
| self.lexstateerrorf = { } |
| for key,ef in lextab._lexstateerrorf.items(): |
| self.lexstateerrorf[key] = fdict[ef] |
| self.begin('INITIAL') |
| |
| # ------------------------------------------------------------ |
| # input() - Push a new string into the lexer |
| # ------------------------------------------------------------ |
| def input(self,s): |
| if not (isinstance(s,util.bytes_type) or isinstance(s, util.string_type)): |
| raise ValueError("Expected a string") |
| self.lexdata = s |
| self.lexpos = 0 |
| self.lexlen = len(s) |
| |
| # ------------------------------------------------------------ |
| # begin() - Changes the lexing state |
| # ------------------------------------------------------------ |
| def begin(self,state): |
| if state not in self.lexstatere: |
| raise ValueError("Undefined state") |
| self.lexre = self.lexstatere[state] |
| self.lexretext = self.lexstateretext[state] |
| self.lexignore = self.lexstateignore.get(state,"") |
| self.lexerrorf = self.lexstateerrorf.get(state,None) |
| self.lexstate = state |
| |
| # ------------------------------------------------------------ |
| # push_state() - Changes the lexing state and saves old on stack |
| # ------------------------------------------------------------ |
| def push_state(self,state): |
| self.lexstatestack.append(self.lexstate) |
| self.begin(state) |
| |
| # ------------------------------------------------------------ |
| # pop_state() - Restores the previous state |
| # ------------------------------------------------------------ |
| def pop_state(self): |
| self.begin(self.lexstatestack.pop()) |
| |
| # ------------------------------------------------------------ |
| # current_state() - Returns the current lexing state |
| # ------------------------------------------------------------ |
| def current_state(self): |
| return self.lexstate |
| |
| # ------------------------------------------------------------ |
| # skip() - Skip ahead n characters |
| # ------------------------------------------------------------ |
| def skip(self,n): |
| self.lexpos += n |
| |
| # ------------------------------------------------------------ |
| # token() - Return the next token from the Lexer |
| # |
| # Note: This function has been carefully implemented to be as fast |
| # as possible. Don't make changes unless you really know what |
| # you are doing |
| # ------------------------------------------------------------ |
| def token(self): |
| # Make local copies of frequently referenced attributes |
| lexpos = self.lexpos |
| lexlen = self.lexlen |
| lexignore = self.lexignore |
| lexdata = self.lexdata |
| |
| while lexpos < lexlen: |
| # This code provides some short-circuit code for whitespace, tabs, and other ignored characters |
| if lexdata[lexpos] in lexignore: |
| lexpos += 1 |
| continue |
| |
| # Look for a regular expression match |
| for lexre,lexindexfunc in self.lexre: |
| m = lexre.match(lexdata,lexpos) |
| if not m: continue |
| |
| # Set last match in lexer so that rules can access it if they want |
| self.lexmatch = m |
| |
| # Create a token for return |
| tok = LexToken() |
| tok.value = m.group() |
| tok.lineno = self.lineno |
| tok.lexpos = lexpos |
| tok.lexer = self |
| |
| lexpos = m.end() |
| i = m.lastindex |
| func,tok.type = lexindexfunc[i] |
| self.lexpos = lexpos |
| |
| if not func: |
| # If no token type was set, it's an ignored token |
| if tok.type: return tok |
| break |
| |
| # if func not callable, it means it's an ignored token |
| if not isinstance(func, collections.Callable): |
| break |
| |
| # If token is processed by a function, call it |
| newtok = func(tok) |
| |
| # Every function must return a token, if nothing, we just move to next token |
| if not newtok: |
| lexpos = self.lexpos # This is here in case user has updated lexpos. |
| break |
| |
| # Verify type of the token. If not in the token map, raise an error |
| if not self.lexoptimize: |
| if newtok.type not in self.lextokens: |
| raise LexError("%s:%d: Rule '%s' returned an unknown token type '%s'" % ( |
| func.__code__.co_filename, func.__code__.co_firstlineno, |
| func.__name__, newtok.type),lexdata[lexpos:]) |
| |
| return newtok |
| else: |
| # No match, see if in literals |
| if lexdata[lexpos] in self.lexliterals: |
| tok = LexToken() |
| tok.value = lexdata[lexpos] |
| tok.lineno = self.lineno |
| tok.lexer = self |
| tok.type = tok.value |
| tok.lexpos = lexpos |
| self.lexpos = lexpos + 1 |
| return tok |
| |
| # No match. Call t_error() if defined. |
| if self.lexerrorf: |
| tok = LexToken() |
| tok.value = self.lexdata[lexpos:] |
| tok.lineno = self.lineno |
| tok.type = "error" |
| tok.lexer = self |
| tok.lexpos = lexpos |
| self.lexpos = lexpos |
| newtok = self.lexerrorf(tok) |
| if lexpos == self.lexpos: |
| # Error method didn't change text position at all. This is an error. |
| raise LexError("Scanning error. Illegal character '%s'" % (lexdata[lexpos]), lexdata[lexpos:]) |
| lexpos = self.lexpos |
| if not newtok: continue |
| return newtok |
| |
| self.lexpos = lexpos |
| raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos],lexpos), lexdata[lexpos:]) |
| |
| self.lexpos = lexpos + 1 |
| if self.lexdata is None: |
| raise RuntimeError("No input string given with input()") |
| return None |
| |
| # ----------------------------------------------------------------------------- |
| # _validate_file() |
| # |
| # This checks to see if there are duplicated t_rulename() functions or strings |
| # in the parser input file. This is done using a simple regular expression |
| # match on each line in the filename. |
| # ----------------------------------------------------------------------------- |
| |
| def _validate_file(filename): |
| import os.path |
| base,ext = os.path.splitext(filename) |
| if ext != '.py': return 1 # No idea what the file is. Return OK |
| |
| try: |
| f = open(filename) |
| lines = f.readlines() |
| f.close() |
| except IOError: |
| return 1 # Oh well |
| |
| fre = re.compile(r'\s*def\s+(t_[a-zA-Z_0-9]*)\(') |
| sre = re.compile(r'\s*(t_[a-zA-Z_0-9]*)\s*=') |
| counthash = { } |
| linen = 1 |
| noerror = 1 |
| for l in lines: |
| m = fre.match(l) |
| if not m: |
| m = sre.match(l) |
| if m: |
| name = m.group(1) |
| prev = counthash.get(name) |
| if not prev: |
| counthash[name] = linen |
| else: |
| print("%s:%d: Rule %s redefined. Previously defined on line %d" % (filename,linen,name,prev)) |
| noerror = 0 |
| linen += 1 |
| return noerror |
| |
| # ----------------------------------------------------------------------------- |
| # _funcs_to_names() |
| # |
| # Given a list of regular expression functions, this converts it to a list |
| # suitable for output to a table file |
| # ----------------------------------------------------------------------------- |
| |
| def _funcs_to_names(funclist): |
| result = [] |
| for f in funclist: |
| if f and f[0]: |
| result.append((f[0].__name__,f[1])) |
| else: |
| result.append(f) |
| return result |
| |
| # ----------------------------------------------------------------------------- |
| # _names_to_funcs() |
| # |
| # Given a list of regular expression function names, this converts it back to |
| # functions. |
| # ----------------------------------------------------------------------------- |
| |
| def _names_to_funcs(namelist,fdict): |
| result = [] |
| for n in namelist: |
| if n and n[0]: |
| result.append((fdict[n[0]],n[1])) |
| else: |
| result.append(n) |
| return result |
| |
| # ----------------------------------------------------------------------------- |
| # _form_master_re() |
| # |
| # This function takes a list of all of the regex components and attempts to |
| # form the master regular expression. Given limitations in the Python re |
| # module, it may be necessary to break the master regex into separate expressions. |
| # ----------------------------------------------------------------------------- |
| |
| def _form_master_re(relist,reflags,ldict): |
| if not relist: return [] |
| regex = "|".join(relist) |
| try: |
| lexre = re.compile(regex,re.VERBOSE | reflags) |
| |
| # Build the index to function map for the matching engine |
| lexindexfunc = [ None ] * (max(lexre.groupindex.values())+1) |
| for f,i in lexre.groupindex.items(): |
| handle = ldict.get(f,None) |
| if type(handle) in (types.FunctionType, types.MethodType): |
| lexindexfunc[i] = (handle,handle.__name__[2:]) |
| elif handle is not None: |
| # If rule was specified as a string, we build an anonymous |
| # callback function to carry out the action |
| if f.find("ignore_") > 0: |
| lexindexfunc[i] = (None,None) |
| print("IGNORE", f) |
| else: |
| lexindexfunc[i] = (None, f[2:]) |
| |
| return [(lexre,lexindexfunc)],[regex] |
| except Exception as e: |
| m = int(len(relist)/2) |
| if m == 0: m = 1 |
| llist, lre = _form_master_re(relist[:m],reflags,ldict) |
| rlist, rre = _form_master_re(relist[m:],reflags,ldict) |
| return llist+rlist, lre+rre |
| |
| # ----------------------------------------------------------------------------- |
| # def _statetoken(s,names) |
| # |
| # Given a declaration name s of the form "t_" and a dictionary whose keys are |
| # state names, this function returns a tuple (states,tokenname) where states |
| # is a tuple of state names and tokenname is the name of the token. For example, |
| # calling this with s = "t_foo_bar_SPAM" might return (('foo','bar'),'SPAM') |
| # ----------------------------------------------------------------------------- |
| |
| def _statetoken(s,names): |
| nonstate = 1 |
| parts = s.split("_") |
| for i in range(1,len(parts)): |
| if parts[i] not in names and parts[i] != 'ANY': break |
| if i > 1: |
| states = tuple(parts[1:i]) |
| else: |
| states = ('INITIAL',) |
| |
| if 'ANY' in states: |
| states = tuple(names.keys()) |
| |
| tokenname = "_".join(parts[i:]) |
| return (states,tokenname) |
| |
| # ----------------------------------------------------------------------------- |
| # lex(module) |
| # |
| # Build all of the regular expression rules from definitions in the supplied module |
| # ----------------------------------------------------------------------------- |
| def lex(module=None,object=None,debug=0,optimize=0,lextab="lextab",reflags=0,nowarn=0): |
| global lexer |
| ldict = None |
| stateinfo = { 'INITIAL' : 'inclusive'} |
| error = 0 |
| files = { } |
| lexobj = Lexer() |
| lexobj.lexdebug = debug |
| lexobj.lexoptimize = optimize |
| global token,input |
| |
| if nowarn: warn = 0 |
| else: warn = 1 |
| |
| if object: module = object |
| |
| if module: |
| # User supplied a module object. |
| if isinstance(module, types.ModuleType): |
| ldict = module.__dict__ |
| elif isinstance(module, _INSTANCETYPE): |
| _items = [(k,getattr(module,k)) for k in dir(module)] |
| ldict = { } |
| for (i,v) in _items: |
| ldict[i] = v |
| else: |
| raise ValueError("Expected a module or instance") |
| lexobj.lexmodule = module |
| |
| else: |
| # No module given. We might be able to get information from the caller. |
| try: |
| raise RuntimeError |
| except RuntimeError: |
| e,b,t = sys.exc_info() |
| f = t.tb_frame |
| f = f.f_back # Walk out to our calling function |
| ldict = f.f_globals # Grab its globals dictionary |
| |
| if optimize and lextab: |
| try: |
| lexobj.readtab(lextab,ldict) |
| token = lexobj.token |
| input = lexobj.input |
| lexer = lexobj |
| return lexobj |
| |
| except ImportError: |
| pass |
| |
| # Get the tokens, states, and literals variables (if any) |
| if (module and isinstance(module,_INSTANCETYPE)): |
| tokens = getattr(module,"tokens",None) |
| states = getattr(module,"states",None) |
| literals = getattr(module,"literals","") |
| else: |
| tokens = ldict.get("tokens",None) |
| states = ldict.get("states",None) |
| literals = ldict.get("literals","") |
| |
| if not tokens: |
| raise SyntaxError("lex: module does not define 'tokens'") |
| if not (isinstance(tokens,list) or isinstance(tokens,tuple)): |
| raise SyntaxError("lex: tokens must be a list or tuple.") |
| |
| # Build a dictionary of valid token names |
| lexobj.lextokens = { } |
| if not optimize: |
| for n in tokens: |
| if not _is_identifier.match(n): |
| print("lex: Bad token name '%s'" % n) |
| error = 1 |
| if warn and n in lexobj.lextokens: |
| print("lex: Warning. Token '%s' multiply defined." % n) |
| lexobj.lextokens[n] = None |
| else: |
| for n in tokens: lexobj.lextokens[n] = None |
| |
| if debug: |
| print("lex: tokens = '%s'" % list(lexobj.lextokens.keys())) |
| |
| try: |
| for c in literals: |
| if not (isinstance(c,util.bytes_type) or isinstance(c, util.string_type)) or len(c) > 1: |
| print("lex: Invalid literal %s. Must be a single character" % repr(c)) |
| error = 1 |
| continue |
| |
| except TypeError: |
| print("lex: Invalid literals specification. literals must be a sequence of characters.") |
| error = 1 |
| |
| lexobj.lexliterals = literals |
| |
| # Build statemap |
| if states: |
| if not (isinstance(states,tuple) or isinstance(states,list)): |
| print("lex: states must be defined as a tuple or list.") |
| error = 1 |
| else: |
| for s in states: |
| if not isinstance(s,tuple) or len(s) != 2: |
| print("lex: invalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')" % repr(s)) |
| error = 1 |
| continue |
| name, statetype = s |
| if isinstance(name, util.string_type): |
| original_name = name |
| name = util.encode_input(name) |
| if not isinstance(name,util.bytes_type) or len(original_name) != len(name): |
| print("lex: state name %s must be a byte string" % repr(original_name)) |
| error = 1 |
| continue |
| if not (statetype == 'inclusive' or statetype == 'exclusive'): |
| print("lex: state type for state %s must be 'inclusive' or 'exclusive'" % name) |
| error = 1 |
| continue |
| if name in stateinfo: |
| print("lex: state '%s' already defined." % name) |
| error = 1 |
| continue |
| stateinfo[name] = statetype |
| |
| # Get a list of symbols with the t_ or s_ prefix |
| tsymbols = [f for f in ldict.keys() if f[:2] == 't_' ] |
| |
| # Now build up a list of functions and a list of strings |
| |
| funcsym = { } # Symbols defined as functions |
| strsym = { } # Symbols defined as strings |
| toknames = { } # Mapping of symbols to token names |
| |
| for s in stateinfo.keys(): |
| funcsym[s] = [] |
| strsym[s] = [] |
| |
| ignore = { } # Ignore strings by state |
| errorf = { } # Error functions by state |
| |
| if len(tsymbols) == 0: |
| raise SyntaxError("lex: no rules of the form t_rulename are defined.") |
| |
| for f in tsymbols: |
| t = ldict[f] |
| states, tokname = _statetoken(f,stateinfo) |
| toknames[f] = tokname |
| |
| if isinstance(t, collections.Callable): |
| for s in states: funcsym[s].append((f,t)) |
| elif (isinstance(t, util.bytes_type) or isinstance(t,util.string_type)): |
| for s in states: strsym[s].append((f,t)) |
| else: |
| print("lex: %s not defined as a function or string" % f) |
| error = 1 |
| |
| # Sort the functions by line number |
| for f in funcsym.values(): |
| f.sort(key=lambda x: x[1].__code__.co_firstlineno) |
| |
| # Sort the strings by regular expression length |
| for s in strsym.values(): |
| s.sort(key=lambda x: len(x[1])) |
| |
| regexs = { } |
| |
| # Build the master regular expressions |
| for state in stateinfo.keys(): |
| regex_list = [] |
| |
| # Add rules defined by functions first |
| for fname, f in funcsym[state]: |
| line = f.__code__.co_firstlineno |
| file = f.__code__.co_filename |
| files[file] = None |
| tokname = toknames[fname] |
| |
| ismethod = isinstance(f, types.MethodType) |
| |
| if not optimize: |
| nargs = f.__code__.co_argcount |
| if ismethod: |
| reqargs = 2 |
| else: |
| reqargs = 1 |
| if nargs > reqargs: |
| print("%s:%d: Rule '%s' has too many arguments." % (file,line,f.__name__)) |
| error = 1 |
| continue |
| |
| if nargs < reqargs: |
| print("%s:%d: Rule '%s' requires an argument." % (file,line,f.__name__)) |
| error = 1 |
| continue |
| |
| if tokname == 'ignore': |
| print("%s:%d: Rule '%s' must be defined as a string." % (file,line,f.__name__)) |
| error = 1 |
| continue |
| |
| if tokname == 'error': |
| errorf[state] = f |
| continue |
| |
| if f.__doc__: |
| if not optimize: |
| try: |
| c = re.compile("(?P<%s>%s)" % (f.__name__,f.__doc__), re.VERBOSE | reflags) |
| if c.match(""): |
| print("%s:%d: Regular expression for rule '%s' matches empty string." % (file,line,f.__name__)) |
| error = 1 |
| continue |
| except re.error as e: |
| print("%s:%d: Invalid regular expression for rule '%s'. %s" % (file,line,f.__name__,e)) |
| if '#' in f.__doc__: |
| print("%s:%d. Make sure '#' in rule '%s' is escaped with '\\#'." % (file,line, f.__name__)) |
| error = 1 |
| continue |
| |
| if debug: |
| print("lex: Adding rule %s -> '%s' (state '%s')" % (f.__name__,f.__doc__, state)) |
| |
| # Okay. The regular expression seemed okay. Let's append it to the master regular |
| # expression we're building |
| |
| regex_list.append("(?P<%s>%s)" % (f.__name__,f.__doc__)) |
| else: |
| print("%s:%d: No regular expression defined for rule '%s'" % (file,line,f.__name__)) |
| |
| # Now add all of the simple rules |
| for name,r in strsym[state]: |
| tokname = toknames[name] |
| |
| if tokname == 'ignore': |
| ignore[state] = r |
| continue |
| |
| if not optimize: |
| if tokname == 'error': |
| raise SyntaxError("lex: Rule '%s' must be defined as a function" % name) |
| error = 1 |
| continue |
| |
| if tokname not in lexobj.lextokens and tokname.find("ignore_") < 0: |
| print("lex: Rule '%s' defined for an unspecified token %s." % (name,tokname)) |
| error = 1 |
| continue |
| try: |
| c = re.compile("(?P<%s>%s)" % (name,r),re.VERBOSE | reflags) |
| if (c.match("")): |
| print("lex: Regular expression for rule '%s' matches empty string." % name) |
| error = 1 |
| continue |
| except re.error as e: |
| print("lex: Invalid regular expression for rule '%s'. %s" % (name,e)) |
| if '#' in r: |
| print("lex: Make sure '#' in rule '%s' is escaped with '\\#'." % name) |
| |
| error = 1 |
| continue |
| if debug: |
| print("lex: Adding rule %s -> '%s' (state '%s')" % (name,r,state)) |
| |
| regex_list.append("(?P<%s>%s)" % (name,r)) |
| |
| if not regex_list: |
| print("lex: No rules defined for state '%s'" % state) |
| error = 1 |
| |
| regexs[state] = regex_list |
| |
| |
| if not optimize: |
| for f in files.keys(): |
| if not _validate_file(f): |
| error = 1 |
| |
| if error: |
| raise SyntaxError("lex: Unable to build lexer.") |
| |
| # From this point forward, we're reasonably confident that we can build the lexer. |
| # No more errors will be generated, but there might be some warning messages. |
| |
| # Build the master regular expressions |
| |
| for state in regexs.keys(): |
| lexre, re_text = _form_master_re(regexs[state],reflags,ldict) |
| lexobj.lexstatere[state] = lexre |
| lexobj.lexstateretext[state] = re_text |
| if debug: |
| for i in range(len(re_text)): |
| print("lex: state '%s'. regex[%d] = '%s'" % (state, i, re_text[i])) |
| |
| # For inclusive states, we need to add the INITIAL state |
| for state,type in stateinfo.items(): |
| if state != "INITIAL" and type == 'inclusive': |
| lexobj.lexstatere[state].extend(lexobj.lexstatere['INITIAL']) |
| lexobj.lexstateretext[state].extend(lexobj.lexstateretext['INITIAL']) |
| |
| lexobj.lexstateinfo = stateinfo |
| lexobj.lexre = lexobj.lexstatere["INITIAL"] |
| lexobj.lexretext = lexobj.lexstateretext["INITIAL"] |
| |
| # Set up ignore variables |
| lexobj.lexstateignore = ignore |
| lexobj.lexignore = lexobj.lexstateignore.get("INITIAL","") |
| |
| # Set up error functions |
| lexobj.lexstateerrorf = errorf |
| lexobj.lexerrorf = errorf.get("INITIAL",None) |
| if warn and not lexobj.lexerrorf: |
| print("lex: Warning. no t_error rule is defined.") |
| |
| # Check state information for ignore and error rules |
| for s,stype in stateinfo.items(): |
| if stype == 'exclusive': |
| if warn and s not in errorf: |
| print("lex: Warning. no error rule is defined for exclusive state '%s'" % s) |
| if warn and s not in ignore and lexobj.lexignore: |
| print("lex: Warning. no ignore rule is defined for exclusive state '%s'" % s) |
| elif stype == 'inclusive': |
| if s not in errorf: |
| errorf[s] = errorf.get("INITIAL",None) |
| if s not in ignore: |
| ignore[s] = ignore.get("INITIAL","") |
| |
| |
| # Create global versions of the token() and input() functions |
| token = lexobj.token |
| input = lexobj.input |
| lexer = lexobj |
| |
| # If in optimize mode, we write the lextab |
| if lextab and optimize: |
| lexobj.writetab(lextab) |
| |
| return lexobj |
| |
| # ----------------------------------------------------------------------------- |
| # runmain() |
| # |
| # This runs the lexer as a main program |
| # ----------------------------------------------------------------------------- |
| |
| def runmain(lexer=None,data=None): |
| if not data: |
| try: |
| filename = sys.argv[1] |
| f = open(filename) |
| data = f.read() |
| f.close() |
| except IndexError: |
| print("Reading from standard input (type EOF to end):") |
| data = sys.stdin.read() |
| |
| if lexer: |
| _input = lexer.input |
| else: |
| _input = input |
| _input(data) |
| if lexer: |
| _token = lexer.token |
| else: |
| _token = token |
| |
| while 1: |
| tok = _token() |
| if not tok: break |
| print("(%s,%r,%d,%d)" % (tok.type, tok.value, tok.lineno,tok.lexpos)) |
| |
| |
| # ----------------------------------------------------------------------------- |
| # @TOKEN(regex) |
| # |
| # This decorator function can be used to set the regex expression on a function |
| # when its docstring might need to be set in an alternative way |
| # ----------------------------------------------------------------------------- |
| |
| def TOKEN(r): |
| def set_doc(f): |
| f.__doc__ = r |
| return f |
| return set_doc |
| |
| # Alternative spelling of the TOKEN decorator |
| Token = TOKEN |
| |
