| /* |
| * [The "BSD licence"] |
| * Copyright (c) 2005-2008 Terence Parr |
| * All rights reserved. |
| * |
| * Conversion to C#: |
| * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc. |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions |
| * are met: |
| * 1. Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * 2. Redistributions in binary form must reproduce the above copyright |
| * notice, this list of conditions and the following disclaimer in the |
| * documentation and/or other materials provided with the distribution. |
| * 3. The name of the author may not be used to endorse or promote products |
| * derived from this software without specific prior written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
| * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
| * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
| * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| namespace Antlr.Runtime |
| { |
| using System.Collections.Generic; |
| using CLSCompliant = System.CLSCompliantAttribute; |
| using IndexOutOfRangeException = System.IndexOutOfRangeException; |
| using StringBuilder = System.Text.StringBuilder; |
| |
| /** Buffer all input tokens but do on-demand fetching of new tokens from |
| * lexer. Useful when the parser or lexer has to set context/mode info before |
| * proper lexing of future tokens. The ST template parser needs this, |
| * for example, because it has to constantly flip back and forth between |
| * inside/output templates. E.g., <names:{hi, <it>}> has to parse names |
| * as part of an expression but "hi, <it>" as a nested template. |
| * |
| * You can't use this stream if you pass whitespace or other off-channel |
| * tokens to the parser. The stream can't ignore off-channel tokens. |
| * (UnbufferedTokenStream is the same way.) |
| * |
| * This is not a subclass of UnbufferedTokenStream because I don't want |
| * to confuse small moving window of tokens it uses for the full buffer. |
| */ |
| [System.Serializable] |
| public class BufferedTokenStream : ITokenStream, ITokenStreamInformation |
| { |
| private ITokenSource _tokenSource; |
| |
| /** Record every single token pulled from the source so we can reproduce |
| * chunks of it later. The buffer in LookaheadStream overlaps sometimes |
| * as its moving window moves through the input. This list captures |
| * everything so we can access complete input text. |
| */ |
| [CLSCompliant(false)] |
| protected List<IToken> _tokens = new List<IToken>(100); |
| |
| /** Track the last mark() call result value for use in rewind(). */ |
| private int _lastMarker; |
| |
| /** The index into the tokens list of the current token (next token |
| * to consume). tokens[p] should be LT(1). p=-1 indicates need |
| * to initialize with first token. The ctor doesn't get a token. |
| * First call to LT(1) or whatever gets the first token and sets p=0; |
| */ |
| [CLSCompliant(false)] |
| protected int _p = -1; |
| |
| public BufferedTokenStream() |
| { |
| } |
| |
| public BufferedTokenStream(ITokenSource tokenSource) |
| { |
| this._tokenSource = tokenSource; |
| } |
| |
| public virtual ITokenSource TokenSource |
| { |
| get |
| { |
| return _tokenSource; |
| } |
| set |
| { |
| this._tokenSource = value; |
| _tokens.Clear(); |
| _p = -1; |
| } |
| } |
| |
| public virtual int Index |
| { |
| get |
| { |
| return _p; |
| } |
| } |
| |
| /// <summary> |
| /// How deep have we gone? |
| /// </summary> |
| public virtual int Range |
| { |
| get; |
| protected set; |
| } |
| |
| public virtual int Count |
| { |
| get |
| { |
| return _tokens.Count; |
| } |
| } |
| |
| public virtual string SourceName |
| { |
| get |
| { |
| return _tokenSource.SourceName; |
| } |
| } |
| |
| public virtual IToken LastToken |
| { |
| get |
| { |
| return LB(1); |
| } |
| } |
| |
| public virtual IToken LastRealToken |
| { |
| get |
| { |
| int i = 0; |
| IToken token; |
| do |
| { |
| i++; |
| token = LB(i); |
| } while (token != null && token.Line <= 0); |
| |
| return token; |
| } |
| } |
| |
| public virtual int MaxLookBehind |
| { |
| get |
| { |
| return int.MaxValue; |
| } |
| } |
| |
| public virtual int Mark() |
| { |
| if (_p == -1) |
| Setup(); |
| _lastMarker = Index; |
| return _lastMarker; |
| } |
| |
| public virtual void Release(int marker) |
| { |
| // no resources to release |
| } |
| |
| public virtual void Rewind(int marker) |
| { |
| Seek(marker); |
| } |
| |
| public virtual void Rewind() |
| { |
| Seek(_lastMarker); |
| } |
| |
| public virtual void Reset() |
| { |
| _p = 0; |
| _lastMarker = 0; |
| } |
| |
| public virtual void Seek(int index) |
| { |
| _p = index; |
| } |
| |
| /** Move the input pointer to the next incoming token. The stream |
| * must become active with LT(1) available. consume() simply |
| * moves the input pointer so that LT(1) points at the next |
| * input symbol. Consume at least one token. |
| * |
| * Walk past any token not on the channel the parser is listening to. |
| */ |
| public virtual void Consume() |
| { |
| if (_p == -1) |
| Setup(); |
| _p++; |
| Sync(_p); |
| } |
| |
| /** Make sure index i in tokens has a token. */ |
| protected virtual void Sync(int i) |
| { |
| int n = i - _tokens.Count + 1; // how many more elements we need? |
| if (n > 0) |
| Fetch(n); |
| } |
| |
| /** add n elements to buffer */ |
| protected virtual void Fetch(int n) |
| { |
| for (int i = 0; i < n; i++) |
| { |
| IToken t = TokenSource.NextToken(); |
| t.TokenIndex = _tokens.Count; |
| _tokens.Add(t); |
| if (t.Type == CharStreamConstants.EndOfFile) |
| break; |
| } |
| } |
| |
| public virtual IToken Get(int i) |
| { |
| if (i < 0 || i >= _tokens.Count) |
| { |
| throw new IndexOutOfRangeException("token index " + i + " out of range 0.." + (_tokens.Count - 1)); |
| } |
| return _tokens[i]; |
| } |
| |
| #if false // why is this different from GetTokens(start, count) ? |
| /// <summary> |
| /// Get all tokens from start..(start+count-1) inclusively |
| /// </summary> |
| public virtual List<IToken> Get(int start, int count) |
| { |
| if (start < 0) |
| throw new ArgumentOutOfRangeException("start"); |
| if (count < 0) |
| throw new ArgumentOutOfRangeException("count"); |
| if (start + count >= _tokens.Count) |
| throw new ArgumentException(); |
| |
| if (_p == -1) |
| Setup(); |
| |
| List<IToken> subset = new List<IToken>(count); |
| for (int i = 0; i < count; i++) |
| { |
| IToken token = _tokens[i]; |
| if (token.Type == TokenTypes.EndOfFile) |
| break; |
| |
| subset.Add(token); |
| } |
| |
| return subset; |
| } |
| #endif |
| |
| public virtual int LA(int i) |
| { |
| return LT(i).Type; |
| } |
| |
| protected virtual IToken LB(int k) |
| { |
| if ((_p - k) < 0) |
| return null; |
| |
| return _tokens[_p - k]; |
| } |
| |
| public virtual IToken LT(int k) |
| { |
| if (_p == -1) |
| Setup(); |
| if (k == 0) |
| return null; |
| if (k < 0) |
| return LB(-k); |
| |
| int i = _p + k - 1; |
| Sync(i); |
| if (i >= _tokens.Count) |
| { |
| // EOF must be last token |
| return _tokens[_tokens.Count - 1]; |
| } |
| |
| if (i > Range) |
| Range = i; |
| |
| return _tokens[_p + k - 1]; |
| } |
| |
| protected virtual void Setup() |
| { |
| Sync(0); |
| _p = 0; |
| } |
| |
| public virtual List<IToken> GetTokens() |
| { |
| return _tokens; |
| } |
| |
| public virtual List<IToken> GetTokens(int start, int stop) |
| { |
| return GetTokens(start, stop, default(BitSet)); |
| } |
| |
| /** Given a start and stop index, return a List of all tokens in |
| * the token type BitSet. Return null if no tokens were found. This |
| * method looks at both on and off channel tokens. |
| */ |
| public virtual List<IToken> GetTokens(int start, int stop, BitSet types) |
| { |
| if (_p == -1) |
| Setup(); |
| if (stop >= _tokens.Count) |
| stop = _tokens.Count - 1; |
| if (start < 0) |
| start = 0; |
| if (start > stop) |
| return null; |
| |
| // list = tokens[start:stop]:{Token t, t.getType() in types} |
| List<IToken> filteredTokens = new List<IToken>(); |
| for (int i = start; i <= stop; i++) |
| { |
| IToken t = _tokens[i]; |
| if (types == null || types.Member(t.Type)) |
| { |
| filteredTokens.Add(t); |
| } |
| } |
| if (filteredTokens.Count == 0) |
| { |
| filteredTokens = null; |
| } |
| return filteredTokens; |
| } |
| |
| public virtual List<IToken> GetTokens(int start, int stop, IEnumerable<int> types) |
| { |
| return GetTokens(start, stop, new BitSet(types)); |
| } |
| |
| public virtual List<IToken> GetTokens(int start, int stop, int ttype) |
| { |
| return GetTokens(start, stop, BitSet.Of(ttype)); |
| } |
| |
| public override string ToString() |
| { |
| if (_p == -1) |
| Setup(); |
| |
| Fill(); |
| return ToString(0, _tokens.Count - 1); |
| } |
| |
| public virtual string ToString(int start, int stop) |
| { |
| if (start < 0 || stop < 0) |
| return null; |
| if (_p == -1) |
| Setup(); |
| if (stop >= _tokens.Count) |
| stop = _tokens.Count - 1; |
| |
| StringBuilder buf = new StringBuilder(); |
| for (int i = start; i <= stop; i++) |
| { |
| IToken t = _tokens[i]; |
| if (t.Type == CharStreamConstants.EndOfFile) |
| break; |
| buf.Append(t.Text); |
| } |
| |
| return buf.ToString(); |
| } |
| |
| public virtual string ToString(IToken start, IToken stop) |
| { |
| if (start != null && stop != null) |
| { |
| return ToString(start.TokenIndex, stop.TokenIndex); |
| } |
| return null; |
| } |
| |
| public virtual void Fill() |
| { |
| if (_p == -1) |
| Setup(); |
| |
| if (_tokens[_p].Type == CharStreamConstants.EndOfFile) |
| return; |
| |
| int i = _p + 1; |
| Sync(i); |
| while (_tokens[i].Type != CharStreamConstants.EndOfFile) |
| { |
| i++; |
| Sync(i); |
| } |
| } |
| } |
| } |
