app/src/processing/app/syntax/TokenMarker.java - platform/external/arduino-ide - Git at Google

 /*
  * TokenMarker.java - Generic token marker
  * Copyright (C) 1998, 1999 Slava Pestov
  *
  * You may use and modify this package for any purpose. Redistribution is
  * permitted, in both source and binary form, provided that this notice
  * remains intact in all source distributions of this package.
  */

 package processing.app.syntax;

 import javax.swing.text.Segment;

 /**
  * A token marker that splits lines of text into tokens. Each token carries
  * a length field and an indentification tag that can be mapped to a color
  * for painting that token.<p>
  *
  * For performance reasons, the linked list of tokens is reused after each
  * line is tokenized. Therefore, the return value of <code>markTokens</code>
  * should only be used for immediate painting. Notably, it cannot be
  * cached.
  *
  * @author Slava Pestov
  */
 public abstract class TokenMarker
 {
         /**
          * A wrapper for the lower-level <code>markTokensImpl</code> method
          * that is called to split a line up into tokens.
          * @param line The line
          * @param lineIndex The line number
          */
         public Token markTokens(Segment line, int lineIndex)
         {
                 if(lineIndex >= length)
                 {
                         throw new IllegalArgumentException("Tokenizing invalid line: "
                                 + lineIndex);
                 }

                 lastToken = null;

                 LineInfo info = lineInfo[lineIndex];
                 LineInfo prev;
                 if(lineIndex == 0)
                         prev = null;
                 else
                         prev = lineInfo[lineIndex - 1];

                 byte oldToken = info.token;
                 byte token = markTokensImpl(prev == null ?
                         Token.NULL : prev.token,line,lineIndex);

                 info.token = token;

                 /*
                  * This is a foul hack. It stops nextLineRequested
                  * from being cleared if the same line is marked twice.
                  *
                  * Why is this necessary? It's all JEditTextArea's fault.
                  * When something is inserted into the text, firing a
                  * document event, the insertUpdate() method shifts the
                  * caret (if necessary) by the amount inserted.
                  *
                  * All caret movement is handled by the select() method,
                  * which eventually pipes the new position to scrollTo()
                  * and calls repaint().
                  *
                  * Note that at this point in time, the new line hasn't
                  * yet been painted; the caret is moved first.
                  *
                  * scrollTo() calls offsetToX(), which tokenizes the line
                  * unless it is being called on the last line painted
                  * (in which case it uses the text area's painter cached
                  * token list). What scrollTo() does next is irrelevant.
                  *
                  * After scrollTo() has done it's job, repaint() is
                  * called, and eventually we end up in paintLine(), whose
                  * job is to paint the changed line. It, too, calls
                  * markTokens().
                  *
                  * The problem was that if the line started a multiline
                  * token, the first markTokens() (done in offsetToX())
                  * would set nextLineRequested (because the line end
                  * token had changed) but the second would clear it
                  * (because the line was the same that time) and therefore
                  * paintLine() would never know that it needed to repaint
                  * subsequent lines.
                  *
                  * This bug took me ages to track down, that's why I wrote
                  * all the relevant info down so that others wouldn't
                  * duplicate it.
                  */
                  if(!(lastLine == lineIndex && nextLineRequested))
                         nextLineRequested = (oldToken != token);

                 lastLine = lineIndex;

                 addToken(0,Token.END);

                 return firstToken;
         }

         /**
          * An abstract method that splits a line up into tokens. It
          * should parse the line, and call <code>addToken()</code> to
          * add syntax tokens to the token list. Then, it should return
          * the initial token type for the next line.<p>
          *
          * For example if the current line contains the start of a
          * multiline comment that doesn't end on that line, this method
          * should return the comment token type so that it continues on
          * the next line.
          *
          * @param token The initial token type for this line
          * @param line The line to be tokenized
          * @param lineIndex The index of the line in the document,
          * starting at 0
          * @return The initial token type for the next line
          */
         protected abstract byte markTokensImpl(byte token, Segment line,
                 int lineIndex);

         /**
          * Returns if the token marker supports tokens that span multiple
          * lines. If this is true, the object using this token marker is
          * required to pass all lines in the document to the
          * <code>markTokens()</code> method (in turn).<p>
          *
          * The default implementation returns true; it should be overridden
          * to return false on simpler token markers for increased speed.
          */
         public boolean supportsMultilineTokens()
         {
                 return true;
         }

         /**
          * Informs the token marker that lines have been inserted into
          * the document. This inserts a gap in the <code>lineInfo</code>
          * array.
          * @param index The first line number
          * @param lines The number of lines
          */
         public void insertLines(int index, int lines)
         {
                 if(lines <= 0)
                         return;
                 length += lines;
                 ensureCapacity(length);
                 int len = index + lines;
                 System.arraycopy(lineInfo,index,lineInfo,len,
                         lineInfo.length - len);

                 for(int i = index + lines - 1; i >= index; i--)
                 {
                         lineInfo[i] = new LineInfo();
                 }
         }

         /**
          * Informs the token marker that line have been deleted from
          * the document. This removes the lines in question from the
          * <code>lineInfo</code> array.
          * @param index The first line number
          * @param lines The number of lines
          */
         public void deleteLines(int index, int lines)
         {
                 if (lines <= 0)
                         return;
                 int len = index + lines;
                 length -= lines;
                 System.arraycopy(lineInfo,len,lineInfo,
                         index,lineInfo.length - len);
         }

         /**
          * Returns the number of lines in this token marker.
          */
         public int getLineCount()
         {
                 return length;
         }

         /**
          * Returns true if the next line should be repainted. This
          * will return true after a line has been tokenized that starts
          * a multiline token that continues onto the next line.
          */
         public boolean isNextLineRequested()
         {
                 return nextLineRequested;
         }

         // protected members

         /**
          * The first token in the list. This should be used as the return
          * value from <code>markTokens()</code>.
          */
         protected Token firstToken;

         /**
          * The last token in the list. New tokens are added here.
          * This should be set to null before a new line is to be tokenized.
          */
         protected Token lastToken;

         /**
          * An array for storing information about lines. It is enlarged and
          * shrunk automatically by the <code>insertLines()</code> and
          * <code>deleteLines()</code> methods.
          */
         protected LineInfo[] lineInfo;

         /**
          * The number of lines in the model being tokenized. This can be
          * less than the length of the <code>lineInfo</code> array.
          */
         protected int length;

         /**
          * The last tokenized line.
          */
         protected int lastLine;

         /**
          * True if the next line should be painted.
          */
         protected boolean nextLineRequested;

         /**
          * Creates a new <code>TokenMarker</code>. This DOES NOT create
          * a lineInfo array; an initial call to <code>insertLines()</code>
          * does that.
          */
         protected TokenMarker()
         {
                 lastLine = -1;
         }

         /**
          * Ensures that the <code>lineInfo</code> array can contain the
          * specified index. This enlarges it if necessary. No action is
          * taken if the array is large enough already.<p>
          *
          * It should be unnecessary to call this under normal
          * circumstances; <code>insertLine()</code> should take care of
          * enlarging the line info array automatically.
          *
          * @param index The array index
          */
         protected void ensureCapacity(int index)
         {
                 if(lineInfo == null)
                         lineInfo = new LineInfo[index + 1];
                 else if(lineInfo.length <= index)
                 {
                         LineInfo[] lineInfoN = new LineInfo[(index + 1) * 2];
                         System.arraycopy(lineInfo,0,lineInfoN,0,
                                          lineInfo.length);
                         lineInfo = lineInfoN;
                 }
         }

         /**
          * Adds a token to the token list.
          * @param length The length of the token
          * @param id The id of the token
          */
         protected void addToken(int length, byte id)
         {
                 if(id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST)
                         throw new InternalError("Invalid id: " + id);

                 if(length == 0 && id != Token.END)
                         return;

                 if(firstToken == null)
                 {
                         firstToken = new Token(length,id);
                         lastToken = firstToken;
                 }
                 else if(lastToken == null)
                 {
                         lastToken = firstToken;
                         firstToken.length = length;
                         firstToken.id = id;
                 }
                 else if(lastToken.next == null)
                 {
                         lastToken.next = new Token(length,id);
                         lastToken = lastToken.next;
                 }
                 else
                 {
                         lastToken = lastToken.next;
                         lastToken.length = length;
                         lastToken.id = id;
                 }
         }

         /**
          * Inner class for storing information about tokenized lines.
          */
         public class LineInfo
         {
                 /**
                  * Creates a new LineInfo object with token = Token.NULL
                  * and obj = null.
                  */
                 public LineInfo()
                 {
                 }

                 /**
                  * Creates a new LineInfo object with the specified
                  * parameters.
                  */
                 public LineInfo(byte token, Object obj)
                 {
                         this.token = token;
                         this.obj = obj;
                 }

                 /**
                  * The id of the last token of the line.
                  */
                 public byte token;

                 /**
                  * This is for use by the token marker implementations
                  * themselves. It can be used to store anything that
                  * is an object and that needs to exist on a per-line
                  * basis.
                  */
                 public Object obj;
         }
 }
	/*
	* TokenMarker.java - Generic token marker
	* Copyright (C) 1998, 1999 Slava Pestov
	*
	* You may use and modify this package for any purpose. Redistribution is
	* permitted, in both source and binary form, provided that this notice
	* remains intact in all source distributions of this package.
	*/

	package processing.app.syntax;

	import javax.swing.text.Segment;

	/**
	* A token marker that splits lines of text into tokens. Each token carries
	* a length field and an indentification tag that can be mapped to a color
	* for painting that token.<p>
	*
	* For performance reasons, the linked list of tokens is reused after each
	* line is tokenized. Therefore, the return value of <code>markTokens</code>
	* should only be used for immediate painting. Notably, it cannot be
	* cached.
	*
	* @author Slava Pestov
	*/
	public abstract class TokenMarker
	{
	/**
	* A wrapper for the lower-level <code>markTokensImpl</code> method
	* that is called to split a line up into tokens.
	* @param line The line
	* @param lineIndex The line number
	*/
	public Token markTokens(Segment line, int lineIndex)
	{
	if(lineIndex >= length)
	{
	throw new IllegalArgumentException("Tokenizing invalid line: "
	+ lineIndex);
	}

	lastToken = null;

	LineInfo info = lineInfo[lineIndex];
	LineInfo prev;
	if(lineIndex == 0)
	prev = null;
	else
	prev = lineInfo[lineIndex - 1];

	byte oldToken = info.token;
	byte token = markTokensImpl(prev == null ?
	Token.NULL : prev.token,line,lineIndex);

	info.token = token;

	/*
	* This is a foul hack. It stops nextLineRequested
	* from being cleared if the same line is marked twice.
	*
	* Why is this necessary? It's all JEditTextArea's fault.
	* When something is inserted into the text, firing a
	* document event, the insertUpdate() method shifts the
	* caret (if necessary) by the amount inserted.
	*
	* All caret movement is handled by the select() method,
	* which eventually pipes the new position to scrollTo()
	* and calls repaint().
	*
	* Note that at this point in time, the new line hasn't
	* yet been painted; the caret is moved first.
	*
	* scrollTo() calls offsetToX(), which tokenizes the line
	* unless it is being called on the last line painted
	* (in which case it uses the text area's painter cached
	* token list). What scrollTo() does next is irrelevant.
	*
	* After scrollTo() has done it's job, repaint() is
	* called, and eventually we end up in paintLine(), whose
	* job is to paint the changed line. It, too, calls
	* markTokens().
	*
	* The problem was that if the line started a multiline
	* token, the first markTokens() (done in offsetToX())
	* would set nextLineRequested (because the line end
	* token had changed) but the second would clear it
	* (because the line was the same that time) and therefore
	* paintLine() would never know that it needed to repaint
	* subsequent lines.
	*
	* This bug took me ages to track down, that's why I wrote
	* all the relevant info down so that others wouldn't
	* duplicate it.
	*/
	if(!(lastLine == lineIndex && nextLineRequested))
	nextLineRequested = (oldToken != token);

	lastLine = lineIndex;

	addToken(0,Token.END);

	return firstToken;
	}

	/**
	* An abstract method that splits a line up into tokens. It
	* should parse the line, and call <code>addToken()</code> to
	* add syntax tokens to the token list. Then, it should return
	* the initial token type for the next line.<p>
	*
	* For example if the current line contains the start of a
	* multiline comment that doesn't end on that line, this method
	* should return the comment token type so that it continues on
	* the next line.
	*
	* @param token The initial token type for this line
	* @param line The line to be tokenized
	* @param lineIndex The index of the line in the document,
	* starting at 0
	* @return The initial token type for the next line
	*/
	protected abstract byte markTokensImpl(byte token, Segment line,
	int lineIndex);

	/**
	* Returns if the token marker supports tokens that span multiple
	* lines. If this is true, the object using this token marker is
	* required to pass all lines in the document to the
	* <code>markTokens()</code> method (in turn).<p>
	*
	* The default implementation returns true; it should be overridden
	* to return false on simpler token markers for increased speed.
	*/
	public boolean supportsMultilineTokens()
	{
	return true;
	}

	/**
	* Informs the token marker that lines have been inserted into
	* the document. This inserts a gap in the <code>lineInfo</code>
	* array.
	* @param index The first line number
	* @param lines The number of lines
	*/
	public void insertLines(int index, int lines)
	{
	if(lines <= 0)
	return;
	length += lines;
	ensureCapacity(length);
	int len = index + lines;
	System.arraycopy(lineInfo,index,lineInfo,len,
	lineInfo.length - len);

	for(int i = index + lines - 1; i >= index; i--)
	{
	lineInfo[i] = new LineInfo();
	}
	}

	/**
	* Informs the token marker that line have been deleted from
	* the document. This removes the lines in question from the
	* <code>lineInfo</code> array.
	* @param index The first line number
	* @param lines The number of lines
	*/
	public void deleteLines(int index, int lines)
	{
	if (lines <= 0)
	return;
	int len = index + lines;
	length -= lines;
	System.arraycopy(lineInfo,len,lineInfo,
	index,lineInfo.length - len);
	}

	/**
	* Returns the number of lines in this token marker.
	*/
	public int getLineCount()
	{
	return length;
	}

	/**
	* Returns true if the next line should be repainted. This
	* will return true after a line has been tokenized that starts
	* a multiline token that continues onto the next line.
	*/
	public boolean isNextLineRequested()
	{
	return nextLineRequested;
	}

	// protected members

	/**
	* The first token in the list. This should be used as the return
	* value from <code>markTokens()</code>.
	*/
	protected Token firstToken;

	/**
	* The last token in the list. New tokens are added here.
	* This should be set to null before a new line is to be tokenized.
	*/
	protected Token lastToken;

	/**
	* An array for storing information about lines. It is enlarged and
	* shrunk automatically by the <code>insertLines()</code> and
	* <code>deleteLines()</code> methods.
	*/
	protected LineInfo[] lineInfo;

	/**
	* The number of lines in the model being tokenized. This can be
	* less than the length of the <code>lineInfo</code> array.
	*/
	protected int length;

	/**
	* The last tokenized line.
	*/
	protected int lastLine;

	/**
	* True if the next line should be painted.
	*/
	protected boolean nextLineRequested;

	/**
	* Creates a new <code>TokenMarker</code>. This DOES NOT create
	* a lineInfo array; an initial call to <code>insertLines()</code>
	* does that.
	*/
	protected TokenMarker()
	{
	lastLine = -1;
	}

	/**
	* Ensures that the <code>lineInfo</code> array can contain the
	* specified index. This enlarges it if necessary. No action is
	* taken if the array is large enough already.<p>
	*
	* It should be unnecessary to call this under normal
	* circumstances; <code>insertLine()</code> should take care of
	* enlarging the line info array automatically.
	*
	* @param index The array index
	*/
	protected void ensureCapacity(int index)
	{
	if(lineInfo == null)
	lineInfo = new LineInfo[index + 1];
	else if(lineInfo.length <= index)
	{
	LineInfo[] lineInfoN = new LineInfo[(index + 1) * 2];
	System.arraycopy(lineInfo,0,lineInfoN,0,
	lineInfo.length);
	lineInfo = lineInfoN;
	}
	}

	/**
	* Adds a token to the token list.
	* @param length The length of the token
	* @param id The id of the token
	*/
	protected void addToken(int length, byte id)
	{
	if(id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST)
	throw new InternalError("Invalid id: " + id);

	if(length == 0 && id != Token.END)
	return;

	if(firstToken == null)
	{
	firstToken = new Token(length,id);
	lastToken = firstToken;
	}
	else if(lastToken == null)
	{
	lastToken = firstToken;
	firstToken.length = length;
	firstToken.id = id;
	}
	else if(lastToken.next == null)
	{
	lastToken.next = new Token(length,id);
	lastToken = lastToken.next;
	}
	else
	{
	lastToken = lastToken.next;
	lastToken.length = length;
	lastToken.id = id;
	}
	}

	/**
	* Inner class for storing information about tokenized lines.
	*/
	public class LineInfo
	{
	/**
	* Creates a new LineInfo object with token = Token.NULL
	* and obj = null.
	*/
	public LineInfo()
	{
	}

	/**
	* Creates a new LineInfo object with the specified
	* parameters.
	*/
	public LineInfo(byte token, Object obj)
	{
	this.token = token;
	this.obj = obj;
	}

	/**
	* The id of the last token of the line.
	*/
	public byte token;

	/**
	* This is for use by the token marker implementations
	* themselves. It can be used to store anything that
	* is an object and that needs to exist on a per-line
	* basis.
	*/
	public Object obj;
	}
	}