| /// \file |
| /// Default implementation of CommonTokenStream |
| /// |
| |
| // [The "BSD licence"] |
| // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC |
| // http://www.temporal-wave.com |
| // http://www.linkedin.com/in/jimidle |
| // |
| // All rights reserved. |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions |
| // are met: |
| // 1. Redistributions of source code must retain the above copyright |
| // notice, this list of conditions and the following disclaimer. |
| // 2. Redistributions in binary form must reproduce the above copyright |
| // notice, this list of conditions and the following disclaimer in the |
| // documentation and/or other materials provided with the distribution. |
| // 3. The name of the author may not be used to endorse or promote products |
| // derived from this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
| // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
| // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
| // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| #include <antlr3tokenstream.h> |
| |
| #ifdef ANTLR3_WINDOWS |
| #pragma warning( disable : 4100 ) |
| #endif |
| |
| // COMMON_TOKEN_STREAM API |
| // |
| static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel); |
| static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype); |
| static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard); |
| static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts); |
| static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); |
| static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types); |
| static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list); |
| static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type); |
| static void reset (pANTLR3_COMMON_TOKEN_STREAM cts); |
| |
| // TOKEN_STREAM API |
| // |
| static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); |
| static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); |
| static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i); |
| static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts); |
| static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource); |
| static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts); |
| static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); |
| static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop); |
| static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger); |
| |
| // INT STREAM API |
| // |
| static void consume (pANTLR3_INT_STREAM is); |
| static void dbgConsume (pANTLR3_INT_STREAM is); |
| static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); |
| static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); |
| static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is); |
| static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is); |
| static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark); |
| static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is); |
| static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is); |
| static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); |
| static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); |
| static void rewindLast (pANTLR3_INT_STREAM is); |
| static void dbgRewindLast (pANTLR3_INT_STREAM is); |
| static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); |
| static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); |
| static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is); |
| static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream); |
| static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream); |
| |
| // Helpers |
| // |
| static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream); |
| static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); |
| static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); |
| static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); |
| |
| ANTLR3_API pANTLR3_TOKEN_STREAM |
| antlr3TokenStreamNew() |
| { |
| pANTLR3_TOKEN_STREAM stream; |
| |
| // Memory for the interface structure |
| // |
| stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM)); |
| |
| if (stream == NULL) |
| { |
| return NULL; |
| } |
| |
| // Install basic API |
| // |
| stream->free = antlr3TokenStreamFree; |
| |
| |
| return stream; |
| } |
| |
| static void |
| antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream) |
| { |
| ANTLR3_FREE(stream); |
| } |
| |
| static void |
| antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream) |
| { |
| // We only free up our subordinate interfaces if they belong |
| // to us, otherwise we let whoever owns them deal with them. |
| // |
| if (stream->tstream->super == stream) |
| { |
| if (stream->tstream->istream->super == stream->tstream) |
| { |
| stream->tstream->istream->free(stream->tstream->istream); |
| stream->tstream->istream = NULL; |
| } |
| stream->tstream->free(stream->tstream); |
| } |
| |
| // Now we free our own resources |
| // |
| if (stream->tokens != NULL) |
| { |
| stream->tokens->free(stream->tokens); |
| stream->tokens = NULL; |
| } |
| if (stream->discardSet != NULL) |
| { |
| stream->discardSet->free(stream->discardSet); |
| stream->discardSet = NULL; |
| } |
| if (stream->channelOverrides != NULL) |
| { |
| stream->channelOverrides->free(stream->channelOverrides); |
| stream->channelOverrides = NULL; |
| } |
| |
| // Free our memory now |
| // |
| ANTLR3_FREE(stream); |
| } |
| |
| // Reset a token stream so it can be used again and can reuse it's |
| // resources. |
| // |
| static void |
| reset (pANTLR3_COMMON_TOKEN_STREAM cts) |
| { |
| |
| // Free any resources that ar most like specifc to the |
| // run we just did. |
| // |
| if (cts->discardSet != NULL) |
| { |
| cts->discardSet->free(cts->discardSet); |
| cts->discardSet = NULL; |
| } |
| if (cts->channelOverrides != NULL) |
| { |
| cts->channelOverrides->free(cts->channelOverrides); |
| cts->channelOverrides = NULL; |
| } |
| |
| // Now, if there were any existing tokens in the stream, |
| // then we just reset the vector count so that it starts |
| // again. We must traverse the entries unfortunately as |
| // there may be free pointers for custom token types and |
| // so on. However that is just a quick NULL check on the |
| // vector entries. |
| // |
| if (cts->tokens != NULL) |
| { |
| cts->tokens->clear(cts->tokens); |
| } |
| else |
| { |
| /* Install the token tracking tables |
| */ |
| cts->tokens = antlr3VectorNew(0); |
| } |
| |
| // Reset to defaults |
| // |
| cts->discardOffChannel = ANTLR3_FALSE; |
| cts->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; |
| cts->p = -1; |
| } |
| |
| ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM |
| antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger) |
| { |
| pANTLR3_COMMON_TOKEN_STREAM stream; |
| |
| // Create a standard token stream |
| // |
| stream = antlr3CommonTokenStreamSourceNew(hint, source); |
| |
| // Install the debugger object |
| // |
| stream->tstream->debugger = debugger; |
| |
| // Override standard token stream methods with debugging versions |
| // |
| stream->tstream->initialStreamState = ANTLR3_FALSE; |
| |
| stream->tstream->_LT = dbgTokLT; |
| |
| stream->tstream->istream->consume = dbgConsume; |
| stream->tstream->istream->_LA = dbgLA; |
| stream->tstream->istream->mark = dbgMark; |
| stream->tstream->istream->rewind = dbgRewindStream; |
| stream->tstream->istream->rewindLast = dbgRewindLast; |
| stream->tstream->istream->seek = dbgSeek; |
| |
| return stream; |
| } |
| |
| ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM |
| antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source) |
| { |
| pANTLR3_COMMON_TOKEN_STREAM stream; |
| |
| stream = antlr3CommonTokenStreamNew(hint); |
| |
| stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; |
| |
| stream->channelOverrides = NULL; |
| stream->discardSet = NULL; |
| stream->discardOffChannel = ANTLR3_FALSE; |
| |
| stream->tstream->setTokenSource(stream->tstream, source); |
| |
| stream->free = antlr3CTSFree; |
| return stream; |
| } |
| |
| ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM |
| antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint) |
| { |
| pANTLR3_COMMON_TOKEN_STREAM stream; |
| |
| /* Memory for the interface structure |
| */ |
| stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM)); |
| |
| if (stream == NULL) |
| { |
| return NULL; |
| } |
| |
| /* Create space for the token stream interface |
| */ |
| stream->tstream = antlr3TokenStreamNew(); |
| stream->tstream->super = stream; |
| |
| /* Create space for the INT_STREAM interfacce |
| */ |
| stream->tstream->istream = antlr3IntStreamNew(); |
| stream->tstream->istream->super = (stream->tstream); |
| stream->tstream->istream->type = ANTLR3_TOKENSTREAM; |
| |
| /* Install the token tracking tables |
| */ |
| stream->tokens = antlr3VectorNew(0); |
| |
| /* Defaults |
| */ |
| stream->p = -1; |
| |
| /* Install the common token stream API |
| */ |
| stream->setTokenTypeChannel = setTokenTypeChannel; |
| stream->discardTokenType = discardTokenType; |
| stream->discardOffChannelToks = discardOffChannel; |
| stream->getTokens = getTokens; |
| stream->getTokenRange = getTokenRange; |
| stream->getTokensSet = getTokensSet; |
| stream->getTokensList = getTokensList; |
| stream->getTokensType = getTokensType; |
| stream->reset = reset; |
| |
| /* Install the token stream API |
| */ |
| stream->tstream->_LT = tokLT; |
| stream->tstream->get = get; |
| stream->tstream->getTokenSource = getTokenSource; |
| stream->tstream->setTokenSource = setTokenSource; |
| stream->tstream->toString = toString; |
| stream->tstream->toStringSS = toStringSS; |
| stream->tstream->toStringTT = toStringTT; |
| stream->tstream->setDebugListener = setDebugListener; |
| |
| /* Install INT_STREAM interface |
| */ |
| stream->tstream->istream->_LA = _LA; |
| stream->tstream->istream->mark = mark; |
| stream->tstream->istream->release = release; |
| stream->tstream->istream->size = size; |
| stream->tstream->istream->index = tindex; |
| stream->tstream->istream->rewind = rewindStream; |
| stream->tstream->istream->rewindLast= rewindLast; |
| stream->tstream->istream->seek = seek; |
| stream->tstream->istream->consume = consume; |
| stream->tstream->istream->getSourceName = getSourceName; |
| |
| return stream; |
| } |
| |
| // Install a debug listener adn switch to debug mode methods |
| // |
| static void |
| setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger) |
| { |
| // Install the debugger object |
| // |
| ts->debugger = debugger; |
| |
| // Override standard token stream methods with debugging versions |
| // |
| ts->initialStreamState = ANTLR3_FALSE; |
| |
| ts->_LT = dbgTokLT; |
| |
| ts->istream->consume = dbgConsume; |
| ts->istream->_LA = dbgLA; |
| ts->istream->mark = dbgMark; |
| ts->istream->rewind = dbgRewindStream; |
| ts->istream->rewindLast = dbgRewindLast; |
| ts->istream->seek = dbgSeek; |
| } |
| |
| /** Get the ith token from the current position 1..n where k=1 is the |
| * first symbol of lookahead. |
| */ |
| static pANTLR3_COMMON_TOKEN |
| tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) |
| { |
| ANTLR3_INT32 i; |
| ANTLR3_INT32 n; |
| pANTLR3_COMMON_TOKEN_STREAM cts; |
| |
| cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; |
| |
| if (k < 0) |
| { |
| return LB(cts, -k); |
| } |
| |
| if (cts->p == -1) |
| { |
| fillBuffer(cts); |
| } |
| |
| // Here we used to check for k == 0 and return 0, but this seems |
| // a superfluous check to me. LT(k=0) is therefore just undefined |
| // and we won't waste the clock cycles on the check |
| // |
| |
| if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize) |
| { |
| pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); |
| |
| teof->setStartIndex (teof, ts->istream->index (ts->istream)); |
| teof->setStopIndex (teof, ts->istream->index (ts->istream)); |
| return teof; |
| } |
| |
| i = cts->p; |
| n = 1; |
| |
| /* Need to find k good tokens, skipping ones that are off channel |
| */ |
| while ( n < k) |
| { |
| /* Skip off-channel tokens */ |
| i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */ |
| n++; |
| } |
| if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize) |
| { |
| pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); |
| |
| teof->setStartIndex (teof, ts->istream->index(ts->istream)); |
| teof->setStopIndex (teof, ts->istream->index(ts->istream)); |
| return teof; |
| } |
| |
| // Here the token must be in the input vector. Rather then incur |
| // function call penalty, we just return the pointer directly |
| // from the vector |
| // |
| return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; |
| //return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i); |
| } |
| |
| /// Debug only method to flag consumption of initial off-channel |
| /// tokens in the input stream |
| /// |
| static void |
| consumeInitialHiddenTokens(pANTLR3_INT_STREAM is) |
| { |
| ANTLR3_MARKER first; |
| ANTLR3_INT32 i; |
| pANTLR3_TOKEN_STREAM ts; |
| |
| ts = (pANTLR3_TOKEN_STREAM) is->super; |
| first = is->index(is); |
| |
| for (i=0; i<first; i++) |
| { |
| ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i)); |
| } |
| |
| ts->initialStreamState = ANTLR3_FALSE; |
| |
| } |
| |
| /// As per the normal tokLT but sends information to the debugger |
| /// |
| static pANTLR3_COMMON_TOKEN |
| dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) |
| { |
| if (ts->initialStreamState == ANTLR3_TRUE) |
| { |
| consumeInitialHiddenTokens(ts->istream); |
| } |
| return tokLT(ts, k); |
| } |
| |
| #ifdef ANTLR3_WINDOWS |
| /* When fully optimized VC7 complains about non reachable code. |
| * Not yet sure if this is an optimizer bug, or a bug in the flow analysis |
| */ |
| #pragma warning( disable : 4702 ) |
| #endif |
| |
| static pANTLR3_COMMON_TOKEN |
| LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k) |
| { |
| ANTLR3_INT32 i; |
| ANTLR3_INT32 n; |
| |
| if (cts->p == -1) |
| { |
| fillBuffer(cts); |
| } |
| if (k == 0) |
| { |
| return NULL; |
| } |
| if ((cts->p - k) < 0) |
| { |
| return NULL; |
| } |
| |
| i = cts->p; |
| n = 1; |
| |
| /* Need to find k good tokens, going backwards, skipping ones that are off channel |
| */ |
| while (n <= (ANTLR3_INT32) k) |
| { |
| /* Skip off-channel tokens |
| */ |
| |
| i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */ |
| n++; |
| } |
| if (i < 0) |
| { |
| return NULL; |
| } |
| // Here the token must be in the input vector. Rather then incut |
| // function call penalty, we jsut return the pointer directly |
| // from the vector |
| // |
| return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; |
| } |
| |
| static pANTLR3_COMMON_TOKEN |
| get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i) |
| { |
| pANTLR3_COMMON_TOKEN_STREAM cts; |
| |
| cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; |
| |
| return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */ |
| } |
| |
| static pANTLR3_TOKEN_SOURCE |
| getTokenSource (pANTLR3_TOKEN_STREAM ts) |
| { |
| return ts->tokenSource; |
| } |
| |
| static void |
| setTokenSource ( pANTLR3_TOKEN_STREAM ts, |
| pANTLR3_TOKEN_SOURCE tokenSource) |
| { |
| ts->tokenSource = tokenSource; |
| } |
| |
| static pANTLR3_STRING |
| toString (pANTLR3_TOKEN_STREAM ts) |
| { |
| pANTLR3_COMMON_TOKEN_STREAM cts; |
| |
| cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; |
| |
| if (cts->p == -1) |
| { |
| fillBuffer(cts); |
| } |
| |
| return ts->toStringSS(ts, 0, ts->istream->size(ts->istream)); |
| } |
| |
| static pANTLR3_STRING |
| toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) |
| { |
| pANTLR3_STRING string; |
| pANTLR3_TOKEN_SOURCE tsource; |
| pANTLR3_COMMON_TOKEN tok; |
| ANTLR3_UINT32 i; |
| pANTLR3_COMMON_TOKEN_STREAM cts; |
| |
| cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; |
| |
| if (cts->p == -1) |
| { |
| fillBuffer(cts); |
| } |
| if (stop >= ts->istream->size(ts->istream)) |
| { |
| stop = ts->istream->size(ts->istream) - 1; |
| } |
| |
| /* Who is giving us these tokens? |
| */ |
| tsource = ts->getTokenSource(ts); |
| |
| if (tsource != NULL && cts->tokens != NULL) |
| { |
| /* Finally, let's get a string |
| */ |
| string = tsource->strFactory->newRaw(tsource->strFactory); |
| |
| for (i = start; i <= stop; i++) |
| { |
| tok = ts->get(ts, i); |
| if (tok != NULL) |
| { |
| string->appendS(string, tok->getText(tok)); |
| } |
| } |
| |
| return string; |
| } |
| return NULL; |
| |
| } |
| |
| static pANTLR3_STRING |
| toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop) |
| { |
| if (start != NULL && stop != NULL) |
| { |
| return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop)); |
| } |
| else |
| { |
| return NULL; |
| } |
| } |
| |
| /** Move the input pointer to the next incoming token. The stream |
| * must become active with LT(1) available. consume() simply |
| * moves the input pointer so that LT(1) points at the next |
| * input symbol. Consume at least one token. |
| * |
| * Walk past any token not on the channel the parser is listening to. |
| */ |
| static void |
| consume (pANTLR3_INT_STREAM is) |
| { |
| pANTLR3_COMMON_TOKEN_STREAM cts; |
| pANTLR3_TOKEN_STREAM ts; |
| |
| ts = (pANTLR3_TOKEN_STREAM) is->super; |
| cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; |
| |
| if ((ANTLR3_UINT32)cts->p < cts->tokens->count) |
| { |
| cts->p++; |
| cts->p = skipOffTokenChannels(cts, cts->p); |
| } |
| } |
| |
| |
| /// As per ordinary consume but notifies the debugger about hidden |
| /// tokens and so on. |
| /// |
| static void |
| dbgConsume (pANTLR3_INT_STREAM is) |
| { |
| pANTLR3_TOKEN_STREAM ts; |
| ANTLR3_MARKER a; |
| ANTLR3_MARKER b; |
| pANTLR3_COMMON_TOKEN t; |
| |
| ts = (pANTLR3_TOKEN_STREAM) is->super; |
| |
| if (ts->initialStreamState == ANTLR3_TRUE) |
| { |
| consumeInitialHiddenTokens(is); |
| } |
| |
| a = is->index(is); // Where are we right now? |
| t = ts->_LT(ts, 1); // Current token from stream |
| |
| consume(is); // Standard consumer |
| |
| b = is->index(is); // Where are we after consuming 1 on channel token? |
| |
| ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token |
| |
| if (b>a+1) |
| { |
| // The standard consume caused the index to advance by more than 1, |
| // which can only happen if it skipped some off-channel tokens. |
| // we need to tell the debugger about those tokens. |
| // |
| ANTLR3_MARKER i; |
| |
| for (i = a+1; i<b; i++) |
| { |
| ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i)); |
| } |
| |
| } |
| } |
| |
| /** A simple filter mechanism whereby you can tell this token stream |
| * to force all tokens of type ttype to be on channel. For example, |
| * when interpreting, we cannot execute actions so we need to tell |
| * the stream to force all WS and NEWLINE to be a different, ignored, |
| * channel. |
| */ |
| static void |
| setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel) |
| { |
| if (tokenStream->channelOverrides == NULL) |
| { |
| tokenStream->channelOverrides = antlr3ListNew(10); |
| } |
| |
| /* We add one to the channel so we can distinguish NULL as being no entry in the |
| * table for a particular token type. |
| */ |
| tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL); |
| } |
| |
| static void |
| discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype) |
| { |
| if (tokenStream->discardSet == NULL) |
| { |
| tokenStream->discardSet = antlr3ListNew(31); |
| } |
| |
| /* We add one to the channel so we can distinguish NULL as being no entry in the |
| * table for a particular token type. We could use bitsets for this I suppose too. |
| */ |
| tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL); |
| } |
| |
| static void |
| discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard) |
| { |
| tokenStream->discardOffChannel = discard; |
| } |
| |
| static pANTLR3_VECTOR |
| getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream) |
| { |
| if (tokenStream->p == -1) |
| { |
| fillBuffer(tokenStream); |
| } |
| |
| return tokenStream->tokens; |
| } |
| |
| static pANTLR3_LIST |
| getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) |
| { |
| return tokenStream->getTokensSet(tokenStream, start, stop, NULL); |
| } |
| /** Given a start and stop index, return a List of all tokens in |
| * the token type BitSet. Return null if no tokens were found. This |
| * method looks at both on and off channel tokens. |
| */ |
| static pANTLR3_LIST |
| getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types) |
| { |
| pANTLR3_LIST filteredList; |
| ANTLR3_UINT32 i; |
| ANTLR3_UINT32 n; |
| pANTLR3_COMMON_TOKEN tok; |
| |
| if (tokenStream->p == -1) |
| { |
| fillBuffer(tokenStream); |
| } |
| if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream)) |
| { |
| stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream); |
| } |
| if (start > stop) |
| { |
| return NULL; |
| } |
| |
| /* We have the range set, now we need to iterate through the |
| * installed tokens and create a new list with just the ones we want |
| * in it. We are just moving pointers about really. |
| */ |
| filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream)); |
| |
| for (i = start, n = 0; i<= stop; i++) |
| { |
| tok = tokenStream->tstream->get(tokenStream->tstream, i); |
| |
| if ( types == NULL |
| || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE) |
| ) |
| { |
| filteredList->put(filteredList, n++, (void *)tok, NULL); |
| } |
| } |
| |
| /* Did we get any then? |
| */ |
| if (filteredList->size(filteredList) == 0) |
| { |
| filteredList->free(filteredList); |
| filteredList = NULL; |
| } |
| |
| return filteredList; |
| } |
| |
| static pANTLR3_LIST |
| getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list) |
| { |
| pANTLR3_BITSET bitSet; |
| pANTLR3_LIST newlist; |
| |
| bitSet = antlr3BitsetList(list->table); |
| |
| newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); |
| |
| bitSet->free(bitSet); |
| |
| return newlist; |
| |
| } |
| |
| static pANTLR3_LIST |
| getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type) |
| { |
| pANTLR3_BITSET bitSet; |
| pANTLR3_LIST newlist; |
| |
| bitSet = antlr3BitsetOf(type, -1); |
| newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); |
| |
| bitSet->free(bitSet); |
| |
| return newlist; |
| } |
| |
| static ANTLR3_UINT32 |
| _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) |
| { |
| pANTLR3_TOKEN_STREAM ts; |
| pANTLR3_COMMON_TOKEN tok; |
| |
| ts = (pANTLR3_TOKEN_STREAM) is->super; |
| |
| tok = ts->_LT(ts, i); |
| |
| if (tok != NULL) |
| { |
| return tok->getType(tok); |
| } |
| else |
| { |
| return ANTLR3_TOKEN_INVALID; |
| } |
| } |
| |
| /// As per _LA() but for debug mode. |
| /// |
| static ANTLR3_UINT32 |
| dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) |
| { |
| pANTLR3_TOKEN_STREAM ts; |
| |
| ts = (pANTLR3_TOKEN_STREAM) is->super; |
| |
| if (ts->initialStreamState == ANTLR3_TRUE) |
| { |
| consumeInitialHiddenTokens(is); |
| } |
| ts->debugger->LT(ts->debugger, i, tokLT(ts, i)); |
| return _LA(is, i); |
| } |
| |
| static ANTLR3_MARKER |
| mark (pANTLR3_INT_STREAM is) |
| { |
| is->lastMarker = is->index(is); |
| return is->lastMarker; |
| } |
| |
| /// As per mark() but with a call to tell the debugger we are doing this |
| /// |
| static ANTLR3_MARKER |
| dbgMark (pANTLR3_INT_STREAM is) |
| { |
| pANTLR3_TOKEN_STREAM ts; |
| |
| ts = (pANTLR3_TOKEN_STREAM) is->super; |
| |
| is->lastMarker = is->index(is); |
| ts->debugger->mark(ts->debugger, is->lastMarker); |
| |
| return is->lastMarker; |
| } |
| |
| static void |
| release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark) |
| { |
| return; |
| } |
| |
| static ANTLR3_UINT32 |
| size (pANTLR3_INT_STREAM is) |
| { |
| pANTLR3_COMMON_TOKEN_STREAM cts; |
| pANTLR3_TOKEN_STREAM ts; |
| |
| if (is->cachedSize > 0) |
| { |
| return is->cachedSize; |
| } |
| ts = (pANTLR3_TOKEN_STREAM) is->super; |
| cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; |
| |
| is->cachedSize = cts->tokens->count; |
| return is->cachedSize; |
| } |
| |
| static ANTLR3_MARKER |
| tindex (pANTLR3_INT_STREAM is) |
| { |
| pANTLR3_COMMON_TOKEN_STREAM cts; |
| pANTLR3_TOKEN_STREAM ts; |
| |
| ts = (pANTLR3_TOKEN_STREAM) is->super; |
| cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; |
| |
| return cts->p; |
| } |
| |
| static void |
| dbgRewindLast (pANTLR3_INT_STREAM is) |
| { |
| pANTLR3_TOKEN_STREAM ts; |
| |
| ts = (pANTLR3_TOKEN_STREAM) is->super; |
| |
| ts->debugger->rewindLast(ts->debugger); |
| |
| is->rewind(is, is->lastMarker); |
| } |
| static void |
| rewindLast (pANTLR3_INT_STREAM is) |
| { |
| is->rewind(is, is->lastMarker); |
| } |
| static void |
| rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) |
| { |
| is->seek(is, (ANTLR3_UINT32)(marker)); |
| } |
| static void |
| dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) |
| { |
| pANTLR3_TOKEN_STREAM ts; |
| |
| ts = (pANTLR3_TOKEN_STREAM) is->super; |
| |
| ts->debugger->rewind(ts->debugger, marker); |
| |
| is->seek(is, (ANTLR3_UINT32)(marker)); |
| } |
| |
| static void |
| seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) |
| { |
| pANTLR3_COMMON_TOKEN_STREAM cts; |
| pANTLR3_TOKEN_STREAM ts; |
| |
| ts = (pANTLR3_TOKEN_STREAM) is->super; |
| cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; |
| |
| cts->p = (ANTLR3_UINT32)index; |
| } |
| static void |
| dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) |
| { |
| // TODO: Implement seek in debugger when Ter adds it to Java |
| // |
| seek(is, index); |
| } |
| ANTLR3_API void |
| fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream) |
| { |
| fillBuffer(tokenStream); |
| } |
| static void |
| fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) { |
| ANTLR3_UINT32 index; |
| pANTLR3_COMMON_TOKEN tok; |
| ANTLR3_BOOLEAN discard; |
| void * channelI; |
| |
| /* Start at index 0 of course |
| */ |
| index = 0; |
| |
| /* Pick out the next token from the token source |
| * Remember we just get a pointer (reference if you like) here |
| * and so if we store it anywhere, we don't set any pointers to auto free it. |
| */ |
| tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); |
| |
| while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF) |
| { |
| discard = ANTLR3_FALSE; /* Assume we are not discarding */ |
| |
| /* I employ a bit of a trick, or perhaps hack here. Rather than |
| * store a pointer to a structure in the override map and discard set |
| * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0 |
| * we can distinguish "not being there" from "being channel or type 0" |
| */ |
| |
| if (tokenStream->discardSet != NULL |
| && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL) |
| { |
| discard = ANTLR3_TRUE; |
| } |
| else if ( tokenStream->discardOffChannel == ANTLR3_TRUE |
| && tok->getChannel(tok) != tokenStream->channel |
| ) |
| { |
| discard = ANTLR3_TRUE; |
| } |
| else if (tokenStream->channelOverrides != NULL) |
| { |
| /* See if this type is in the override map |
| */ |
| channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1); |
| |
| if (channelI != NULL) |
| { |
| /* Override found |
| */ |
| tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1); |
| } |
| } |
| |
| /* If not discarding it, add it to the list at the current index |
| */ |
| if (discard == ANTLR3_FALSE) |
| { |
| /* Add it, indicating that we will delete it and the table should not |
| */ |
| tok->setTokenIndex(tok, index); |
| tokenStream->p++; |
| tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL); |
| index++; |
| } |
| |
| tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); |
| } |
| |
| /* Cache the size so we don't keep doing indirect method calls. We do this as |
| * early as possible so that anything after this may utilize the cached value. |
| */ |
| tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count; |
| |
| /* Set the consume pointer to the first token that is on our channel |
| */ |
| tokenStream->p = 0; |
| tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p); |
| |
| } |
| |
| /// Given a starting index, return the index of the first on-channel |
| /// token. |
| /// |
| static ANTLR3_UINT32 |
| skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) { |
| ANTLR3_INT32 n; |
| pANTLR3_COMMON_TOKEN tok; |
| |
| n = tokenStream->tstream->istream->cachedSize; |
| |
| while (i < n) |
| { |
| tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element; |
| |
| if (tok->channel!= tokenStream->channel) |
| { |
| i++; |
| } |
| else |
| { |
| return i; |
| } |
| } |
| return i; |
| } |
| |
| static ANTLR3_UINT32 |
| skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x) |
| { |
| pANTLR3_COMMON_TOKEN tok; |
| |
| while (x >= 0) |
| { |
| tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element; |
| |
| if ((tok->channel != tokenStream->channel)) |
| { |
| x--; |
| } |
| else |
| { |
| return x; |
| } |
| } |
| return x; |
| } |
| |
| /// Return a string that represents the name assoicated with the input source |
| /// |
| /// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream. |
| /// |
| /// /returns |
| /// /implements ANTLR3_INT_STREAM_struct::getSourceName() |
| /// |
| static pANTLR3_STRING |
| getSourceName (pANTLR3_INT_STREAM is) |
| { |
| // Slightly convoluted as we must trace back to the lexer's input source |
| // via the token source. The streamName that is here is not initialized |
| // because this is a token stream, not a file or string stream, which are the |
| // only things that have a context for a source name. |
| // |
| return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName; |
| } |