runtime/Cpp/include/antlr3tokenstream.inl

ANTLR_BEGIN_NAMESPACE()

template<class ImplTraits>
TokenSource<ImplTraits>::TokenSource()
	:m_eofToken( ImplTraits::CommonTokenType::TOKEN_EOF), 
	m_skipToken( ImplTraits::CommonTokenType::TOKEN_INVALID)
{
}

template<class ImplTraits>
ANTLR_INLINE typename TokenSource<ImplTraits>::CommonTokenType& TokenSource<ImplTraits>::get_eofToken()
{
	return m_eofToken;
}

template<class ImplTraits>
ANTLR_INLINE const typename TokenSource<ImplTraits>::TokenType& TokenSource<ImplTraits>::get_eofToken() const
{
	return m_eofToken;
}

template<class ImplTraits>
ANTLR_INLINE typename TokenSource<ImplTraits>::CommonTokenType& TokenSource<ImplTraits>::get_skipToken()
{
	return m_skipToken;
}

template<class ImplTraits>
ANTLR_INLINE typename TokenSource<ImplTraits>::StringType& TokenSource<ImplTraits>::get_fileName()
{
	return m_fileName;
}

template<class ImplTraits>
ANTLR_INLINE void TokenSource<ImplTraits>::set_fileName( const StringType& fileName )
{
	m_fileName = fileName;
}

template<class ImplTraits>
typename TokenSource<ImplTraits>::LexerType* TokenSource<ImplTraits>::get_super()
{
	return static_cast<LexerType*>(this);
}

template<class ImplTraits>
typename TokenSource<ImplTraits>::TokenType*	TokenSource<ImplTraits>::nextTokenStr()
{
	typedef typename LexerType::RecognizerSharedStateType RecognizerSharedStateType;
	typedef typename LexerType::InputStreamType InputStreamType;
	typedef typename LexerType::IntStreamType IntStreamType;
	LexerType*                  lexer;
    RecognizerSharedStateType*	state;
    InputStreamType*            input;
    IntStreamType*              istream;

    lexer   = this->get_super();
    state   = lexer->get_rec()->get_state();
    input   = lexer->get_input();
    istream = input->get_istream();

    /// Loop until we get a non skipped token or EOF
    ///
    for	(;;)
    {
        // Get rid of any previous token (token factory takes care of
        // any de-allocation when this token is finally used up.
        //
        state->set_token_present(false);
        state->set_error(false);	    // Start out without an exception
        state->set_failed(false);

        // Now call the matching rules and see if we can generate a new token
        //
        for	(;;)
        {
            // Record the start of the token in our input stream.
            //
            state->set_channel( TOKEN_DEFAULT_CHANNEL );
            state->set_tokenStartCharIndex( (ANTLR_MARKER)input->get_nextChar() );
            state->set_tokenStartCharPositionInLine( input->get_charPositionInLine() );
            state->set_tokenStartLine( input->get_line() );
            state->set_text("");

            if  (istream->_LA(1) == ANTLR_CHARSTREAM_EOF)
            {
                // Reached the end of the current stream, nothing more to do if this is
                // the last in the stack.
                //
                TokenType&    teof = m_eofToken;

                teof.set_startIndex(lexer->getCharIndex());
                teof.set_stopIndex(lexer->getCharIndex());
                teof.set_line(lexer->getLine());
                return  &teof;
            }

            state->set_token_present( false );
            state->set_error(false);	    // Start out without an exception
            state->set_failed(false);

            // Call the generated lexer, see if it can get a new token together.
            //
            lexer->mTokens();

            if  (state->get_error()  == true)
            {
                // Recognition exception, report it and try to recover.
                //
                state->set_failed(true);
                lexer->get_rec()->reportError();
                lexer->recover(); 
            }
            else
            {
                if ( !state->get_token_present() )
                {
                    // Emit the real token, which adds it in to the token stream basically
                    //
                    lexer->emit();
                }
                else if	( *(state->get_token()) ==  m_skipToken )
                {
                    // A real token could have been generated, but "Computer say's naaaaah" and it
                    // it is just something we need to skip altogether.
                    //
                    continue;
                }

                // Good token, not skipped, not EOF token
                //
                return  state->get_token();
            }
        }
    }
}

template<class ImplTraits>
typename TokenSource<ImplTraits>::TokenType*  TokenSource<ImplTraits>::nextToken()
{
	return this->nextToken( BoolForwarder<LexerType::IsFiltered>() );
}

template<class ImplTraits>
typename TokenSource<ImplTraits>::CommonTokenType*	TokenSource<ImplTraits>::nextToken( BoolForwarder<true> /*isFiltered*/ )
{
	LexerType*   lexer;
	typename LexerType::RecognizerSharedStateType* state;

	lexer   = this->get_super();
	state	= lexer->get_lexstate();

	/* Get rid of any previous token (token factory takes care of
		* any deallocation when this token is finally used up.
		*/
	state->set_token_present( false );
	state->set_error( false );	    /* Start out without an exception	*/
	state->set_failed(false);

	/* Record the start of the token in our input stream.
		*/
	state->set_tokenStartCharIndex( lexer->index() );
	state->set_tokenStartCharPositionInLine( lexer->getCharPositionInLine() );
	state->set_tokenStartLine( lexer->getLine() );
	state->set_text("");

	/* Now call the matching rules and see if we can generate a new token
		*/
	for	(;;)
	{
		if (lexer->LA(1) == ANTLR_CHARSTREAM_EOF)
		{
			/* Reached the end of the stream, nothing more to do.
				*/
			CommonTokenType&    teof = m_eofToken;

			teof.set_startIndex(lexer->getCharIndex());
			teof.set_stopIndex(lexer->getCharIndex());
			teof.set_line(lexer->getLine());
			return  &teof;
		}

		state->set_token_present(false);
		state->set_error(false);	    /* Start out without an exception	*/

		{
			ANTLR_MARKER   m;

			m	= lexer->get_istream()->mark();
			state->set_backtracking(1);				/* No exceptions */
			state->set_failed(false);

			/* Call the generated lexer, see if it can get a new token together.
				*/
			lexer->mTokens();
    		state->set_backtracking(0);

    		/* mTokens backtracks with synpred at BACKTRACKING==2
				and we set the synpredgate to allow actions at level 1. */

			if(state->get_failed())
			{
				lexer->rewind(m);
				lexer->consume(); //<! advance one char and try again !>
			}
			else
			{
				lexer->emit();					/* Assemble the token and emit it to the stream */
				TokenType* tok = state->get_token();
				return tok;
			}
		}
	}
}

template<class ImplTraits>
typename TokenSource<ImplTraits>::CommonTokenType*	TokenSource<ImplTraits>::nextToken( BoolForwarder<false> /*isFiltered*/ )
{
	// Find the next token in the current stream
	//
	CommonTokenType* tok = this->nextTokenStr();

	// If we got to the EOF token then switch to the previous
	// input stream if there were any and just return the
	// EOF if there are none. We must check the next token
	// in any outstanding input stream we pop into the active
	// role to see if it was sitting at EOF after PUSHing the
	// stream we just consumed, otherwise we will return EOF
	// on the reinstalled input stream, when in actual fact
	// there might be more input streams to POP before the
	// real EOF of the whole logical inptu stream. Hence we
	// use a while loop here until we find somethign in the stream
	// that isn't EOF or we reach the actual end of the last input
	// stream on the stack.
	//
	while(tok->get_type() == CommonTokenType::TOKEN_EOF)
	{
		typename ImplTraits::LexerType*   lexer;
		lexer   = static_cast<typename ImplTraits::LexerType*>( this->get_super() );

		if  ( lexer->get_rec()->get_state()->get_streams().size() > 0)
		{
			// We have another input stream in the stack so we
			// need to revert to it, then resume the loop to check
			// it wasn't sitting at EOF itself.
			//
			lexer->popCharStream();
			tok = this->nextTokenStr();
		}
		else
		{
			// There were no more streams on the input stack
			// so this EOF is the 'real' logical EOF for
			// the input stream. So we just exit the loop and 
			// return the EOF we have found.
			//
			break;
		}
		
	}

	// return whatever token we have, which may be EOF
	//
	return  tok;
}

template<class ImplTraits>
TokenStream<ImplTraits>::TokenStream()
{
	m_tokenSource = NULL;
	m_debugger = NULL;
	m_initialStreamState = false;
}

template<class ImplTraits>
typename TokenStream<ImplTraits>::IntStreamType* TokenStream<ImplTraits>::get_istream()
{
	return this;
}

template<class ImplTraits>
TokenStream<ImplTraits>::TokenStream(TokenSourceType* source, DebugEventListenerType* debugger)
{
	m_initialStreamState = false;
	m_tokenSource = source;
	m_debugger = debugger;
}

template<class ImplTraits>
CommonTokenStream<ImplTraits>::CommonTokenStream(ANTLR_UINT32 , TokenSourceType* source, 
													DebugEventListenerType* debugger)
					: CommonTokenStream<ImplTraits>::BaseType( source, debugger )
{
	m_p = -1;
	m_channel = TOKEN_DEFAULT_CHANNEL;
	m_discardOffChannel = false;
	m_nissued = 0;
}

template<class ImplTraits>
typename CommonTokenStream<ImplTraits>::TokensType& CommonTokenStream<ImplTraits>::get_tokens()
{
	return m_tokens;
}

template<class ImplTraits>
const typename CommonTokenStream<ImplTraits>::TokensType& CommonTokenStream<ImplTraits>::get_tokens() const
{
	return m_tokens;
}

template<class ImplTraits>
typename CommonTokenStream<ImplTraits>::DiscardSetType& CommonTokenStream<ImplTraits>::get_discardSet()
{
	return m_discardSet;
}

template<class ImplTraits>
const typename CommonTokenStream<ImplTraits>::DiscardSetType& CommonTokenStream<ImplTraits>::get_discardSet() const
{
	return m_discardSet;
}

template<class ImplTraits>
ANTLR_INLINE ANTLR_INT32 CommonTokenStream<ImplTraits>::get_p() const
{
	return m_p;
}

template<class ImplTraits>
ANTLR_INLINE void CommonTokenStream<ImplTraits>::set_p( ANTLR_INT32 p )
{
	m_p = p;
}

template<class ImplTraits>
ANTLR_INLINE void CommonTokenStream<ImplTraits>::inc_p()
{
	++m_p;
}

template<class ImplTraits>
ANTLR_INLINE void CommonTokenStream<ImplTraits>::dec_p()
{
	--m_p;
}

template<class ImplTraits>
ANTLR_INLINE ANTLR_MARKER CommonTokenStream<ImplTraits>::index_impl()
{
	return m_p;
}

// Reset a token stream so it can be used again and can reuse it's
// resources.
//
template<class ImplTraits>
void  CommonTokenStream<ImplTraits>::reset()
{
	// Free any resources that ar most like specifc to the
    // run we just did.
    //
	m_discardSet.clear();
	m_channelOverrides.clear();

    // Now, if there were any existing tokens in the stream,
    // then we just reset the vector count so that it starts
    // again. We must traverse the entries unfortunately as
    // there may be free pointers for custom token types and
    // so on. However that is just a quick NULL check on the
    // vector entries.
    //
	m_tokens.clear();

    // Reset to defaults
    //
    m_discardOffChannel  = false;
    m_channel            = ImplTraits::CommonTokenType::TOKEN_DEFAULT_CHANNEL;
    m_p	            = -1;
}

template<class ImplTraits>
void	TokenStream<ImplTraits>::setDebugListener(DebugEventListenerType* debugger)
{
	m_debugger = debugger;
	m_initialStreamState = false;
}

template<class ImplTraits>
const typename TokenStream<ImplTraits>::TokenType*  TokenStream<ImplTraits>::_LT(ANTLR_INT32 k)
{
	ANTLR_INT32    i;
	ANTLR_INT32    n;
	TokenStreamType* cts;

	cts	    = this->get_super();

    if(k < 0)
	{
		return cts->LB(-k);
	}

	ANTLR_INT32 req_idx = cts->get_p() + k - 1;
	ANTLR_INT32 cached_size = static_cast<ANTLR_INT32>(this->get_istream()->get_cachedSize());

	if(	(cts->get_p() == -1) ||
		( ( req_idx >= cached_size ) && ( (cached_size % ImplTraits::TOKEN_FILL_BUFFER_INCREMENT) == 0 ) )
	  )
	{
		cts->fillBuffer();
	}

    // Here we used to check for k == 0 and return 0, but this seems
    // a superfluous check to me. LT(k=0) is therefore just undefined
    // and we won't waste the clock cycles on the check
    //
	cached_size = static_cast<ANTLR_INT32>(this->get_istream()->get_cachedSize());
	if	( req_idx >= cached_size )
	{
		TokenType&    teof = cts->get_tokenSource()->get_eofToken();

		teof.set_startIndex( this->get_istream()->index());
		teof.set_stopIndex( this->get_istream()->index());
		return  &teof;
	}

	i	= cts->get_p();
	n	= 1;

	/* Need to find k good tokens, skipping ones that are off channel
	*/
	while( n < k)
	{
		/* Skip off-channel tokens */
		i = cts->skipOffTokenChannels(i+1); /* leave p on valid token    */
		n++;
	}
	
	if( ( i >= cached_size ) && ( (cached_size % ImplTraits::TOKEN_FILL_BUFFER_INCREMENT) == 0 ) )
	{
		cts->fillBuffer();
	}
	if	( (ANTLR_UINT32) i >= this->get_istream()->get_cachedSize() )
	{
		TokenType&    teof = cts->get_tokenSource()->get_eofToken();

		teof.set_startIndex(this->get_istream()->index());
		teof.set_stopIndex(this->get_istream()->index());
		return  &teof;
	}

	// Here the token must be in the input vector. Rather then incur
	// function call penalty, we just return the pointer directly
	// from the vector
	//
	return cts->getToken(i);
}

template<class ImplTraits>
const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::LB(ANTLR_INT32 k)
{
    ANTLR_INT32 i;
    ANTLR_INT32 n;

    if (m_p == -1)
    {
        this->fillBuffer();
    }
    if (k == 0)
    {
        return NULL;
    }
    if ((m_p - k) < 0)
    {
        return NULL;
    }

    i = m_p;
    n = 1;

    /* Need to find k good tokens, going backwards, skipping ones that are off channel
     */
    while (n <= k)
    {
        /* Skip off-channel tokens
         */

        i = this->skipOffTokenChannelsReverse(i - 1); /* leave p on valid token    */
        n++;
    }
    if (i < 0)
    {
        return NULL;
    }
	
	// Here the token must be in the input vector. Rather then incut
	// function call penalty, we jsut return the pointer directly
	// from the vector
	//
	return this->getToken(i);
}

template<class ImplTraits>
const typename CommonTokenStream<ImplTraits>::TokenType*   CommonTokenStream<ImplTraits>::getToken(ANTLR_MARKER i)
{
	return this->get(i);
}


template<class ImplTraits>
const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::get(ANTLR_MARKER i)
{
	return this->getToken( static_cast<ANTLR_MARKER>(i), 
							BoolForwarder<ImplTraits::TOKENS_ACCESSED_FROM_OWNING_RULE>() );
}

template<class ImplTraits>
const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::getToken( ANTLR_MARKER tok_idx,
															BoolForwarder<true>  /*tokens_accessed_from_owning_rule*/  )
{
	typename TokensType::iterator iter = m_tokens.find(tok_idx);
	if( iter == m_tokens.end() )
	{
		TokenAccessException ex;
		throw ex;
	}
	const TokenType& tok = iter->second;
    return  &tok; 
}

template<class ImplTraits>
const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::getToken( ANTLR_MARKER tok_idx, BoolForwarder<false>  /*tokens_accessed_from_owning_rule*/   )
{
	TokenType& tok = m_tokens.at( static_cast<ANTLR_UINT32>(tok_idx) );
    return  &tok; 
}

template<class ImplTraits>
typename TokenStream<ImplTraits>::TokenSourceType* TokenStream<ImplTraits>::get_tokenSource() const
{
	return m_tokenSource;
}

template<class ImplTraits>
void TokenStream<ImplTraits>::set_tokenSource( TokenSourceType* tokenSource )
{
	m_tokenSource = tokenSource;
}

template<class ImplTraits>
typename TokenStream<ImplTraits>::StringType	TokenStream<ImplTraits>::toString()
{
	TokenStreamType* cts = static_cast<TokenStreamType>(this);

	if	(cts->get_p() == -1)
    {
		cts->fillBuffer();
    }

    return  this->toStringSS(0, this->get_istream()->size());
}

template<class ImplTraits>
typename TokenStream<ImplTraits>::StringType
TokenStream<ImplTraits>::toStringSS(ANTLR_MARKER start, ANTLR_MARKER stop)
{
    StringType string;
    TokenSourceType* tsource;
    const TokenType* tok;
    TokenStreamType* cts;

    cts = this->get_super();

    if (cts->get_p() == -1)
    {
        cts->fillBuffer();
    }
    if (stop >= this->get_istream()->size())
    {
        stop = this->get_istream()->size() - 1;
    }

    /* Who is giving us these tokens?
     */
    tsource = cts->get_tokenSource();

    if (tsource != NULL && !cts->get_tokens().empty() )
    {
        /* Finally, let's get a string
         */
        for (ANTLR_MARKER i = start; i <= stop; i++)
        {
            tok = cts->get(i);
            if (tok != NULL)
            {
                string.append( tok->getText() );
            }
        }

        return string;
    }
    return "";
}

template<class ImplTraits>
typename TokenStream<ImplTraits>::StringType
TokenStream<ImplTraits>::toStringTT(const TokenType* start, const TokenType* stop)
{
	if	(start != NULL && stop != NULL)
	{
		return	this->toStringSS( start->get_tokenIndex(), 
								  stop->get_tokenIndex());
	}
	else
	{
		return	"";
	}
}

/** A simple filter mechanism whereby you can tell this token stream
 *  to force all tokens of type ttype to be on channel.  For example,
 *  when interpreting, we cannot execute actions so we need to tell
 *  the stream to force all WS and NEWLINE to be a different, ignored,
 *  channel.
 */
template<class ImplTraits>
void	CommonTokenStream<ImplTraits>::setTokenTypeChannel ( ANTLR_UINT32 ttype, ANTLR_UINT32 channel)
{
    /* We add one to the channel so we can distinguish NULL as being no entry in the
     * table for a particular token type.
     */
    m_channelOverrides[ttype] = (ANTLR_UINT32)channel + 1;

}

template<class ImplTraits>
void  CommonTokenStream<ImplTraits>::discardTokenType(ANTLR_INT32 ttype)
{
	 /* We add one to the channel so we can distinguish NULL as being no entry in the
     * table for a particular token type. We could use bitsets for this I suppose too.
     */
	m_discardSet.insert(ttype);
}

template<class ImplTraits>
void CommonTokenStream<ImplTraits>::discardOffChannelToks(bool discard)
{
	m_discardOffChannel = discard;
}

template<class ImplTraits>
typename CommonTokenStream<ImplTraits>::TokensType*  CommonTokenStream<ImplTraits>::getTokens()
{
	if	(m_p == -1)
    {
		this->fillBuffer();
    }

    return  &m_tokens;
}

template<class ImplTraits>
void CommonTokenStream<ImplTraits>::getTokenRange(ANTLR_UINT32 start, ANTLR_UINT32 stop, 
																	TokensListType& tokenRange)
{
	return this->getTokensSet(start, stop, NULL, tokenRange);
}

/** Given a start and stop index, return a List of all tokens in
 *  the token type BitSet.  Return null if no tokens were found.  This
 *  method looks at both on and off channel tokens.
 */
template<class ImplTraits>
void
CommonTokenStream<ImplTraits>::getTokensSet(ANTLR_UINT32 start, ANTLR_UINT32 stop, BitsetType* types,
                                                    TokensListType& filteredList )
{
    ANTLR_UINT32	    i;
    ANTLR_UINT32	    n;
    TokenType*	tok;

    if	( m_p == -1)
    {
		this->fillBuffer();
    }
    if	(stop > this->get_istream()->size())
    {
		stop = this->get_istream()->size();
    }
    if	(start > stop)
    {
		return;
    }

    /* We have the range set, now we need to iterate through the
     * installed tokens and create a new list with just the ones we want
     * in it. We are just moving pointers about really.
     */
    for(i = start, n = 0; i<= stop; i++)
    {
		tok = this->get(i);

		if  (	   types == NULL
			|| (types->isMember( tok->get_type() ) == true )
			)
		{
			filteredList.push_back(tok);
		}
	}
    
    return ;
}

template<class ImplTraits>
void
CommonTokenStream<ImplTraits>::getTokensList(ANTLR_UINT32 start, ANTLR_UINT32 stop, 
													const IntListType& list, TokensListType& newlist)
{
    BitsetType*		bitSet;

    bitSet  = Bitset<ImplTraits>::BitsetFromList(list);
    this->getTokensSet(start, stop, bitSet, newlist);
    delete bitSet;
}

template<class ImplTraits>
void 
CommonTokenStream<ImplTraits>::getTokensType(ANTLR_UINT32 start, ANTLR_UINT32 stop, ANTLR_UINT32 type,
                                                  TokensListType& newlist   )
{
    BitsetType*  bitSet;

    bitSet  = BitsetType::BitsetOf(type, -1);
    this->getTokensSet(start, stop, bitSet, newlist);

    delete bitSet;
}

template<class ImplTraits>
void CommonTokenStream<ImplTraits>::fillBufferExt()
{
    this->fillBuffer();
}

template<class ImplTraits>
bool CommonTokenStream<ImplTraits>::hasReachedFillbufferTarget( ANTLR_UINT32 cnt, 
																BoolForwarder<true> )
{
	return ( cnt >= ImplTraits::TOKEN_FILL_BUFFER_INCREMENT );
}

template<class ImplTraits>
bool CommonTokenStream<ImplTraits>::hasReachedFillbufferTarget( ANTLR_UINT32, 
																BoolForwarder<false>  )
{
	return false;
}


template<class ImplTraits>
void CommonTokenStream<ImplTraits>::fillBuffer() 
{
    ANTLR_UINT32 index;
    TokenType* tok;
    bool discard;
    
    /* Start at index 0 of course
     */
	ANTLR_UINT32 cached_p = (m_p < 0) ? 0 : m_p;
    index = m_nissued;
	ANTLR_UINT32 cnt = 0;

    /* Pick out the next token from the token source
     * Remember we just get a pointer (reference if you like) here
     * and so if we store it anywhere, we don't set any pointers to auto free it.
     */
    tok = this->get_tokenSource()->nextToken();

    while ( tok->get_type() != TokenType::TOKEN_EOF )
    {
        discard = false; /* Assume we are not discarding	*/

        /* I employ a bit of a trick, or perhaps hack here. Rather than
         * store a pointer to a structure in the override map and discard set
         * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
         * we can distinguish "not being there" from "being channel or type 0"
         */

        if ( m_discardSet.find(tok->get_type()) != m_discardSet.end() )
        {
            discard = true;
        }
        else if (   m_discardOffChannel == true
                 && tok->get_channel() != m_channel
                 )
        {
            discard = true;
        }
        else if (!m_channelOverrides.empty())
        {
            /* See if this type is in the override map
             */
			typename ChannelOverridesType::iterator iter = m_channelOverrides.find( tok->get_type() + 1 );

            if (iter != m_channelOverrides.end())
            {
                /* Override found
                 */
                tok->set_channel( ANTLR_UINT32_CAST(iter->second) - 1);
            }
        }

        /* If not discarding it, add it to the list at the current index
         */
        if (discard == false)
        {
            /* Add it, indicating that we will delete it and the table should not
             */
            tok->set_tokenIndex(index);
            ++m_p;
            this->insertToken(*tok);
            index++;
			m_nissued++;
			cnt++;
        }

		if( !this->hasReachedFillbufferTarget( cnt, 
						BoolForwarder<ImplTraits::TOKENS_ACCESSED_FROM_OWNING_RULE>()  ) )
			tok = this->get_tokenSource()->nextToken();
		else
			break;
    }

    /* Cache the size so we don't keep doing indirect method calls. We do this as
     * early as possible so that anything after this may utilize the cached value.
     */
    this->get_istream()->set_cachedSize( m_nissued );

    /* Set the consume pointer to the first token that is on our channel, we just read
     */
    m_p = cached_p;
    m_p = this->skipOffTokenChannels( m_p );

}
/// Given a starting index, return the index of the first on-channel
///  token.
///
template<class ImplTraits>
ANTLR_UINT32 CommonTokenStream<ImplTraits>::skipOffTokenChannels(ANTLR_INT32 i)
{
    ANTLR_INT32 n;
    n = this->get_istream()->get_cachedSize();

    while (i < n)
    {
        const TokenType* tok =  this->getToken(i);

        if (tok->get_channel() != m_channel )
        {
            i++;
        }
        else
        {
            return i;
        }
    }
    return i;
}

template<class ImplTraits>
ANTLR_UINT32  CommonTokenStream<ImplTraits>::skipOffTokenChannelsReverse(ANTLR_INT32 x)
{
    while (x >= 0)
    {
        const TokenType* tok =  this->getToken(x);
        
        if( tok->get_channel() != m_channel )
        {
            x--;
        }
        else
        {
            return x;
        }
    }
    return x;
}

template<class ImplTraits>
void CommonTokenStream<ImplTraits>::discardTokens( ANTLR_MARKER start, ANTLR_MARKER stop )
{
	this->discardTokens( start, stop, BoolForwarder< ImplTraits::TOKENS_ACCESSED_FROM_OWNING_RULE >() );
}

template<class ImplTraits>
void CommonTokenStream<ImplTraits>::discardTokens( ANTLR_MARKER start, ANTLR_MARKER stop, 
											BoolForwarder<true>  /*tokens_accessed_from_owning_rule */ )
{
	typename TokensType::iterator iter1 = m_tokens.lower_bound(start);
	typename TokensType::iterator iter2 = m_tokens.upper_bound(stop);
	m_tokens.erase( iter1, iter2 );
}

template<class ImplTraits>
void CommonTokenStream<ImplTraits>::discardTokens( ANTLR_MARKER start, ANTLR_MARKER stop, 
											BoolForwarder<false>  /*tokens_accessed_from_owning_rule*/ )
{
	m_tokens.erase( m_tokens.begin() + start, m_tokens.begin() + stop );
}

template<class ImplTraits>
void CommonTokenStream<ImplTraits>::insertToken( const TokenType& tok )
{
	this->insertToken( tok, BoolForwarder< ImplTraits::TOKENS_ACCESSED_FROM_OWNING_RULE >() );
}

template<class ImplTraits>
void CommonTokenStream<ImplTraits>::insertToken( const TokenType& tok, BoolForwarder<true>  /*tokens_accessed_from_owning_rule*/  )
{
	assert( m_tokens.find( tok.get_index() ) == m_tokens.end() );
	assert( tok.get_index() == m_nissued );
	m_tokens[ tok.get_index() ] = tok;
}

template<class ImplTraits>
void CommonTokenStream<ImplTraits>::insertToken( const TokenType& tok, BoolForwarder<false>  /*tokens_accessed_from_owning_rule*/  )
{
	m_tokens.push_back( tok );
}

template<class ImplTraits>
CommonTokenStream<ImplTraits>::~CommonTokenStream()
{
	m_tokens.clear();
}

ANTLR_END_NAMESPACE()