| /* |
| * NOTE TO JL: The original contains references to 3.5-only stuff! |
| * |
| * [The "BSD licence"] |
| * Copyright (c) 2005-2008 Terence Parr |
| * All rights reserved. |
| * |
| * Conversion to C#: |
| * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc. |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions |
| * are met: |
| * 1. Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * 2. Redistributions in binary form must reproduce the above copyright |
| * notice, this list of conditions and the following disclaimer in the |
| * documentation and/or other materials provided with the distribution. |
| * 3. The name of the author may not be used to endorse or promote products |
| * derived from this software without specific prior written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
| * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
| * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
| * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| namespace Antlr.Runtime { |
| using System.Collections.Generic; |
| |
| using ArgumentOutOfRangeException = System.ArgumentOutOfRangeException; |
| using InvalidOperationException = System.InvalidOperationException; |
| using StringBuilder = System.Text.StringBuilder; |
| |
| /** <summary> |
| * The most common stream of tokens is one where every token is buffered up |
| * and tokens are prefiltered for a certain channel (the parser will only |
| * see these tokens and cannot change the filter channel number during the |
| * parse). |
| * </summary> |
| * |
| * <remarks>TODO: how to access the full token stream? How to track all tokens matched per rule?</remarks> |
| */ |
| [System.Serializable] |
| public class LegacyCommonTokenStream : ITokenStream { |
| [System.NonSerialized] |
| ITokenSource _tokenSource; |
| |
| /** <summary> |
| * Record every single token pulled from the source so we can reproduce |
| * chunks of it later. |
| * </summary> |
| */ |
| protected List<IToken> tokens; |
| |
| /** <summary>Map<tokentype, channel> to override some Tokens' channel numbers</summary> */ |
| protected IDictionary<int, int> channelOverrideMap; |
| |
| /** <summary>Set<tokentype>; discard any tokens with this type</summary> */ |
| protected Dictionary<int, int> discardSet; |
| |
| /** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */ |
| protected int channel = TokenChannels.Default; |
| |
| /** <summary>By default, track all incoming tokens</summary> */ |
| protected bool discardOffChannelTokens = false; |
| |
| /** <summary>Track the last mark() call result value for use in rewind().</summary> */ |
| protected int lastMarker; |
| |
| /** <summary> |
| * The index into the tokens list of the current token (next token |
| * to consume). p==-1 indicates that the tokens list is empty |
| * </summary> |
| */ |
| protected int p = -1; |
| |
| public LegacyCommonTokenStream() { |
| tokens = new List<IToken>(500); |
| } |
| |
| public LegacyCommonTokenStream(ITokenSource tokenSource) |
| : this() { |
| this._tokenSource = tokenSource; |
| } |
| |
| public LegacyCommonTokenStream(ITokenSource tokenSource, int channel) |
| : this(tokenSource) { |
| this.channel = channel; |
| } |
| |
| public virtual int Index { |
| get { |
| return p; |
| } |
| } |
| |
| /// <summary> |
| /// How deep have we gone? |
| /// </summary> |
| public virtual int Range { |
| get; |
| protected set; |
| } |
| |
| /** <summary>Reset this token stream by setting its token source.</summary> */ |
| public virtual void SetTokenSource(ITokenSource tokenSource) { |
| this._tokenSource = tokenSource; |
| tokens.Clear(); |
| p = -1; |
| channel = TokenChannels.Default; |
| } |
| |
| /** <summary> |
| * Load all tokens from the token source and put in tokens. |
| * This is done upon first LT request because you might want to |
| * set some token type / channel overrides before filling buffer. |
| * </summary> |
| */ |
| public virtual void FillBuffer() { |
| // fast return if the buffer is already full |
| if (p != -1) |
| return; |
| |
| int index = 0; |
| IToken t = _tokenSource.NextToken(); |
| while (t != null && t.Type != CharStreamConstants.EndOfFile) { |
| bool discard = false; |
| // is there a channel override for token type? |
| int channelI; |
| if (channelOverrideMap != null && channelOverrideMap.TryGetValue(t.Type, out channelI)) |
| t.Channel = channelI; |
| |
| //if ( channelOverrideMap != null && channelOverrideMap.ContainsKey( t.getType() ) ) |
| //{ |
| // object channelI = channelOverrideMap.get( t.getType() ); |
| // if ( channelI != null ) |
| // { |
| // t.setChannel( (int)channelI ); |
| // } |
| //} |
| if (discardSet != null && |
| discardSet.ContainsKey(t.Type)) { |
| discard = true; |
| } else if (discardOffChannelTokens && t.Channel != this.channel) { |
| discard = true; |
| } |
| if (!discard) { |
| t.TokenIndex = index; |
| tokens.Add(t); |
| index++; |
| } |
| t = _tokenSource.NextToken(); |
| } |
| // leave p pointing at first token on channel |
| p = 0; |
| p = SkipOffTokenChannels(p); |
| } |
| |
| /** <summary> |
| * Move the input pointer to the next incoming token. The stream |
| * must become active with LT(1) available. consume() simply |
| * moves the input pointer so that LT(1) points at the next |
| * input symbol. Consume at least one token. |
| * </summary> |
| * |
| * <remarks> |
| * Walk past any token not on the channel the parser is listening to. |
| * </remarks> |
| */ |
| public virtual void Consume() { |
| if (p < tokens.Count) { |
| p++; |
| p = SkipOffTokenChannels(p); // leave p on valid token |
| } |
| } |
| |
| /** <summary>Given a starting index, return the index of the first on-channel token.</summary> */ |
| protected virtual int SkipOffTokenChannels(int i) { |
| int n = tokens.Count; |
| while (i < n && ((IToken)tokens[i]).Channel != channel) { |
| i++; |
| } |
| return i; |
| } |
| |
| protected virtual int SkipOffTokenChannelsReverse(int i) { |
| while (i >= 0 && ((IToken)tokens[i]).Channel != channel) { |
| i--; |
| } |
| return i; |
| } |
| |
| /** <summary> |
| * A simple filter mechanism whereby you can tell this token stream |
| * to force all tokens of type ttype to be on channel. For example, |
| * when interpreting, we cannot exec actions so we need to tell |
| * the stream to force all WS and NEWLINE to be a different, ignored |
| * channel. |
| * </summary> |
| */ |
| public virtual void SetTokenTypeChannel(int ttype, int channel) { |
| if (channelOverrideMap == null) { |
| channelOverrideMap = new Dictionary<int, int>(); |
| } |
| channelOverrideMap[ttype] = channel; |
| } |
| |
| public virtual void DiscardTokenType(int ttype) { |
| if (discardSet == null) { |
| discardSet = new Dictionary<int, int>(); |
| } |
| discardSet.Add(ttype, ttype); |
| } |
| |
| public virtual void SetDiscardOffChannelTokens(bool discardOffChannelTokens) { |
| this.discardOffChannelTokens = discardOffChannelTokens; |
| } |
| |
| public virtual IList<IToken> GetTokens() { |
| if (p == -1) { |
| FillBuffer(); |
| } |
| return tokens; |
| } |
| |
| public virtual IList<IToken> GetTokens(int start, int stop) { |
| return GetTokens(start, stop, (BitSet)null); |
| } |
| |
| /** <summary> |
| * Given a start and stop index, return a List of all tokens in |
| * the token type BitSet. Return null if no tokens were found. This |
| * method looks at both on and off channel tokens. |
| * </summary> |
| */ |
| public virtual IList<IToken> GetTokens(int start, int stop, BitSet types) { |
| if (p == -1) { |
| FillBuffer(); |
| } |
| if (stop >= tokens.Count) { |
| stop = tokens.Count - 1; |
| } |
| if (start < 0) { |
| start = 0; |
| } |
| if (start > stop) { |
| return null; |
| } |
| |
| // list = tokens[start:stop]:{Token t, t.getType() in types} |
| IList<IToken> filteredTokens = new List<IToken>(); |
| for (int i = start; i <= stop; i++) { |
| IToken t = tokens[i]; |
| if (types == null || types.Member(t.Type)) { |
| filteredTokens.Add(t); |
| } |
| } |
| if (filteredTokens.Count == 0) { |
| filteredTokens = null; |
| } |
| return filteredTokens; |
| } |
| |
| public virtual IList<IToken> GetTokens(int start, int stop, IList<int> types) { |
| return GetTokens(start, stop, new BitSet(types)); |
| } |
| |
| public virtual IList<IToken> GetTokens(int start, int stop, int ttype) { |
| return GetTokens(start, stop, BitSet.Of(ttype)); |
| } |
| |
| /** <summary> |
| * Get the ith token from the current position 1..n where k=1 is the |
| * first symbol of lookahead. |
| * </summary> |
| */ |
| public virtual IToken LT(int k) { |
| if (p == -1) { |
| FillBuffer(); |
| } |
| if (k == 0) { |
| return null; |
| } |
| if (k < 0) { |
| return LB(-k); |
| } |
| //System.out.print("LT(p="+p+","+k+")="); |
| if ((p + k - 1) >= tokens.Count) { |
| return tokens[tokens.Count - 1]; |
| } |
| //System.out.println(tokens.get(p+k-1)); |
| int i = p; |
| int n = 1; |
| // find k good tokens |
| while (n < k) { |
| // skip off-channel tokens |
| i = SkipOffTokenChannels(i + 1); // leave p on valid token |
| n++; |
| } |
| if (i >= tokens.Count) { |
| return tokens[tokens.Count - 1]; |
| } |
| |
| if (i > Range) |
| Range = i; |
| |
| return (IToken)tokens[i]; |
| } |
| |
| /** <summary>Look backwards k tokens on-channel tokens</summary> */ |
| protected virtual IToken LB(int k) { |
| //System.out.print("LB(p="+p+","+k+") "); |
| if (p == -1) { |
| FillBuffer(); |
| } |
| if (k == 0) { |
| return null; |
| } |
| if ((p - k) < 0) { |
| return null; |
| } |
| |
| int i = p; |
| int n = 1; |
| // find k good tokens looking backwards |
| while (n <= k) { |
| // skip off-channel tokens |
| i = SkipOffTokenChannelsReverse(i - 1); // leave p on valid token |
| n++; |
| } |
| if (i < 0) { |
| return null; |
| } |
| return (IToken)tokens[i]; |
| } |
| |
| /** <summary> |
| * Return absolute token i; ignore which channel the tokens are on; |
| * that is, count all tokens not just on-channel tokens. |
| * </summary> |
| */ |
| public virtual IToken Get(int i) { |
| return (IToken)tokens[i]; |
| } |
| |
| #if false |
| /** Get all tokens from start..stop inclusively */ |
| public virtual List<IToken> Get(int start, int count) |
| { |
| if (start < 0) |
| throw new ArgumentOutOfRangeException("start"); |
| if (count < 0) |
| throw new ArgumentOutOfRangeException("count"); |
| |
| if (p == -1) |
| FillBuffer(); |
| |
| return new List<IToken>(tokens.Skip(start).Take(count)); |
| } |
| #endif |
| |
| public virtual int LA(int i) { |
| return LT(i).Type; |
| } |
| |
| public virtual int Mark() { |
| if (p == -1) { |
| FillBuffer(); |
| } |
| lastMarker = Index; |
| return lastMarker; |
| } |
| |
| public virtual void Release(int marker) { |
| // no resources to release |
| } |
| |
| public virtual int Count { |
| get { |
| return tokens.Count; |
| } |
| } |
| |
| public virtual void Rewind(int marker) { |
| Seek(marker); |
| } |
| |
| public virtual void Rewind() { |
| Seek(lastMarker); |
| } |
| |
| public virtual void Reset() { |
| p = 0; |
| lastMarker = 0; |
| } |
| |
| public virtual void Seek(int index) { |
| p = index; |
| } |
| |
| public virtual ITokenSource TokenSource { |
| get { |
| return _tokenSource; |
| } |
| } |
| |
| public virtual string SourceName { |
| get { |
| return TokenSource.SourceName; |
| } |
| } |
| |
| public override string ToString() { |
| if (p == -1) { |
| throw new InvalidOperationException("Buffer is not yet filled."); |
| } |
| return ToString(0, tokens.Count - 1); |
| } |
| |
| public virtual string ToString(int start, int stop) { |
| if (start < 0 || stop < 0) { |
| return null; |
| } |
| if (p == -1) { |
| throw new InvalidOperationException("Buffer is not yet filled."); |
| } |
| if (stop >= tokens.Count) { |
| stop = tokens.Count - 1; |
| } |
| StringBuilder buf = new StringBuilder(); |
| for (int i = start; i <= stop; i++) { |
| IToken t = tokens[i]; |
| buf.Append(t.Text); |
| } |
| return buf.ToString(); |
| } |
| |
| public virtual string ToString(IToken start, IToken stop) { |
| if (start != null && stop != null) { |
| return ToString(start.TokenIndex, stop.TokenIndex); |
| } |
| return null; |
| } |
| } |
| } |