| /* |
| * [The "BSD license"] |
| * Copyright (c) 2011 Terence Parr |
| * All rights reserved. |
| * |
| * Conversion to C#: |
| * Copyright (c) 2011 Sam Harwell, Pixel Mine, Inc. |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions |
| * are met: |
| * 1. Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * 2. Redistributions in binary form must reproduce the above copyright |
| * notice, this list of conditions and the following disclaimer in the |
| * documentation and/or other materials provided with the distribution. |
| * 3. The name of the author may not be used to endorse or promote products |
| * derived from this software without specific prior written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
| * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
| * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
| * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| namespace Antlr.Runtime |
| { |
| using System.Collections.Generic; |
| using System; |
| using System.Text; |
| |
| /** <summary> |
| * A pretty quick CharStream that pulls all data from an array |
| * directly. Every method call counts in the lexer. Java's |
| * strings aren't very good so I'm avoiding. |
| * </summary> |
| */ |
| [System.Serializable] |
| public class FastTokenStream |
| : ITokenStream<SlimToken> |
| { |
| [System.NonSerialized] |
| ITokenSource<SlimToken> _tokenSource; |
| |
| /** <summary> |
| * Record every single token pulled from the source so we can reproduce |
| * chunks of it later. |
| * </summary> |
| */ |
| protected List<SlimToken> tokens; |
| |
| /** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */ |
| protected int channel = TokenChannels.Default; |
| |
| /** <summary>Track the last mark() call result value for use in rewind().</summary> */ |
| protected int lastMarker; |
| |
| /** <summary> |
| * The index into the tokens list of the current token (next token |
| * to consume). p==-1 indicates that the tokens list is empty |
| * </summary> |
| */ |
| protected int p = -1; |
| |
| public FastTokenStream() |
| { |
| tokens = new List<SlimToken>( 500 ); |
| } |
| |
| public FastTokenStream( ITokenSource<SlimToken> tokenSource ) |
| : this() |
| { |
| this._tokenSource = tokenSource; |
| } |
| |
| public FastTokenStream( ITokenSource<SlimToken> tokenSource, int channel ) |
| : this( tokenSource ) |
| { |
| this.channel = channel; |
| } |
| |
| public int Index |
| { |
| get |
| { |
| return p; |
| } |
| } |
| |
| /** <summary>Reset this token stream by setting its token source.</summary> */ |
| public void SetTokenSource( ITokenSource<SlimToken> tokenSource ) |
| { |
| this._tokenSource = tokenSource; |
| tokens.Clear(); |
| p = -1; |
| channel = TokenChannels.Default; |
| } |
| |
| /** <summary> |
| * Load all tokens from the token source and put in tokens. |
| * This is done upon first LT request because you might want to |
| * set some token type / channel overrides before filling buffer. |
| * </summary> |
| */ |
| public void FillBuffer() |
| { |
| // fast return if the buffer is already full |
| if ( p != -1 ) |
| return; |
| |
| int index = 0; |
| SlimToken t = _tokenSource.NextToken(); |
| while ( t.Type != CharStreamConstants.EndOfFile ) |
| { |
| //t.TokenIndex = index; |
| tokens.Add( t ); |
| index++; |
| |
| t = _tokenSource.NextToken(); |
| } |
| // leave p pointing at first token on channel |
| p = 0; |
| p = SkipOffTokenChannels( p ); |
| } |
| |
| /** <summary> |
| * Move the input pointer to the next incoming token. The stream |
| * must become active with LT(1) available. consume() simply |
| * moves the input pointer so that LT(1) points at the next |
| * input symbol. Consume at least one token. |
| * </summary> |
| * |
| * <remarks> |
| * Walk past any token not on the channel the parser is listening to. |
| * </remarks> |
| */ |
| public void Consume() |
| { |
| if ( p < tokens.Count ) |
| { |
| p++; |
| p = SkipOffTokenChannels( p ); // leave p on valid token |
| } |
| } |
| |
| /** <summary>Given a starting index, return the index of the first on-channel token.</summary> */ |
| protected int SkipOffTokenChannels( int i ) |
| { |
| int n = tokens.Count; |
| while ( i < n && tokens[i].Channel != channel ) |
| { |
| i++; |
| } |
| return i; |
| } |
| |
| protected int SkipOffTokenChannelsReverse( int i ) |
| { |
| while ( i >= 0 && tokens[i].Channel != channel ) |
| { |
| i--; |
| } |
| return i; |
| } |
| |
| public IList<SlimToken> GetTokens() |
| { |
| if ( p == -1 ) |
| { |
| FillBuffer(); |
| } |
| return tokens; |
| } |
| |
| public IList<SlimToken> GetTokens( int start, int stop ) |
| { |
| return GetTokens( start, stop, (BitSet)null ); |
| } |
| |
| /** <summary> |
| * Given a start and stop index, return a List of all tokens in |
| * the token type BitSet. Return null if no tokens were found. This |
| * method looks at both on and off channel tokens. |
| * </summary> |
| */ |
| public IList<SlimToken> GetTokens( int start, int stop, BitSet types ) |
| { |
| if ( p == -1 ) |
| { |
| FillBuffer(); |
| } |
| if ( stop >= tokens.Count ) |
| { |
| stop = tokens.Count - 1; |
| } |
| if ( start < 0 ) |
| { |
| start = 0; |
| } |
| if ( start > stop ) |
| { |
| return null; |
| } |
| |
| // list = tokens[start:stop]:{Token t, t.getType() in types} |
| List<SlimToken> filteredTokens = new List<SlimToken>(); |
| for ( int i = start; i <= stop; i++ ) |
| { |
| SlimToken t = tokens[i]; |
| if ( types == null || types.Member( t.Type ) ) |
| { |
| filteredTokens.Add( t ); |
| } |
| } |
| if ( filteredTokens.Count == 0 ) |
| { |
| filteredTokens = null; |
| } |
| return filteredTokens; |
| } |
| |
| public IList<SlimToken> GetTokens( int start, int stop, IList<int> types ) |
| { |
| return GetTokens( start, stop, new BitSet( types ) ); |
| } |
| |
| public IList<SlimToken> GetTokens( int start, int stop, int ttype ) |
| { |
| return GetTokens( start, stop, BitSet.Of( ttype ) ); |
| } |
| |
| /** <summary> |
| * Get the ith token from the current position 1..n where k=1 is the |
| * first symbol of lookahead. |
| * </summary> |
| */ |
| public SlimToken LT( int k ) |
| { |
| if ( p == -1 ) |
| { |
| FillBuffer(); |
| } |
| if ( k == 0 ) |
| { |
| return default( SlimToken ); |
| } |
| if ( k < 0 ) |
| { |
| return LB( -k ); |
| } |
| //System.out.print("LT(p="+p+","+k+")="); |
| if ( ( p + k - 1 ) >= tokens.Count ) |
| { |
| return new SlimToken(TokenTypes.EndOfFile); |
| } |
| //System.out.println(tokens.get(p+k-1)); |
| int i = p; |
| int n = 1; |
| // find k good tokens |
| while ( n < k ) |
| { |
| // skip off-channel tokens |
| i = SkipOffTokenChannels( i + 1 ); // leave p on valid token |
| n++; |
| } |
| if ( i >= tokens.Count ) |
| { |
| return new SlimToken(TokenTypes.EndOfFile); |
| } |
| return tokens[i]; |
| } |
| |
| /** <summary>Look backwards k tokens on-channel tokens</summary> */ |
| protected SlimToken LB( int k ) |
| { |
| //System.out.print("LB(p="+p+","+k+") "); |
| if ( p == -1 ) |
| { |
| FillBuffer(); |
| } |
| if ( k == 0 ) |
| { |
| return default( SlimToken ); |
| } |
| if ( ( p - k ) < 0 ) |
| { |
| return default( SlimToken ); |
| } |
| |
| int i = p; |
| int n = 1; |
| // find k good tokens looking backwards |
| while ( n <= k ) |
| { |
| // skip off-channel tokens |
| i = SkipOffTokenChannelsReverse( i - 1 ); // leave p on valid token |
| n++; |
| } |
| if ( i < 0 ) |
| { |
| return default( SlimToken ); |
| } |
| return tokens[i]; |
| } |
| |
| /** <summary> |
| * Return absolute token i; ignore which channel the tokens are on; |
| * that is, count all tokens not just on-channel tokens. |
| * </summary> |
| */ |
| public SlimToken Get( int i ) |
| { |
| return tokens[i]; |
| } |
| |
| public int LA( int i ) |
| { |
| return LT( i ).Type; |
| } |
| |
| public int Mark() |
| { |
| if ( p == -1 ) |
| { |
| FillBuffer(); |
| } |
| lastMarker = Index; |
| return lastMarker; |
| } |
| |
| public void Release( int marker ) |
| { |
| // no resources to release |
| } |
| |
| public int Count |
| { |
| get |
| { |
| return tokens.Count; |
| } |
| } |
| |
| public void Rewind( int marker ) |
| { |
| Seek( marker ); |
| } |
| |
| public void Rewind() |
| { |
| Seek( lastMarker ); |
| } |
| |
| public void Reset() |
| { |
| p = 0; |
| lastMarker = 0; |
| } |
| |
| public void Seek( int index ) |
| { |
| p = index; |
| } |
| |
| public ITokenSource<SlimToken> TokenSource |
| { |
| get |
| { |
| return _tokenSource; |
| } |
| } |
| |
| public string SourceName |
| { |
| get |
| { |
| return TokenSource.SourceName; |
| } |
| } |
| |
| public override string ToString() |
| { |
| if ( p == -1 ) |
| { |
| throw new InvalidOperationException( "Buffer is not yet filled." ); |
| } |
| return ToString( 0, tokens.Count - 1 ); |
| } |
| |
| public virtual string ToString( int start, int stop ) |
| { |
| if ( start < 0 || stop < 0 ) |
| { |
| return null; |
| } |
| if ( p == -1 ) |
| { |
| throw new InvalidOperationException( "Buffer is not yet filled." ); |
| } |
| if ( stop >= tokens.Count ) |
| { |
| stop = tokens.Count - 1; |
| } |
| StringBuilder buf = new StringBuilder(); |
| for ( int i = start; i <= stop; i++ ) |
| { |
| SlimToken t = tokens[i]; |
| SlimLexer lexer = _tokenSource as SlimLexer; |
| if ( lexer != null ) |
| { |
| SlimStringStream input = lexer.CharStream as SlimStringStream; |
| if ( input != null ) |
| { |
| string text = input.Substring( t.StartIndex, t.StopIndex - t.StartIndex + 1 ); |
| buf.Append( text ); |
| } |
| } |
| } |
| return buf.ToString(); |
| } |
| |
| public virtual string ToString( IToken start, IToken stop ) |
| { |
| if ( start != null && stop != null ) |
| { |
| return ToString( start.TokenIndex, stop.TokenIndex ); |
| } |
| return null; |
| } |
| } |
| } |
