antlr-runtime/antlr-3.4/runtime/C/src/antlr3commontoken.c - toolchain/jack - Git at Google

 /**
  * Contains the default implementation of the common token used within
  * java. Custom tokens should create this structure and then append to it using the
  * custom pointer to install their own structure and API.
  */

 // [The "BSD licence"]
 // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
 // http://www.temporal-wave.com
 // http://www.linkedin.com/in/jimidle
 //
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
 // are met:
 // 1. Redistributions of source code must retain the above copyright
 //    notice, this list of conditions and the following disclaimer.
 // 2. Redistributions in binary form must reproduce the above copyright
 //    notice, this list of conditions and the following disclaimer in the
 //    documentation and/or other materials provided with the distribution.
 // 3. The name of the author may not be used to endorse or promote products
 //    derived from this software without specific prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 #include    <antlr3.h>

 /* Token API
  */
 static  pANTLR3_STRING	getText					(pANTLR3_COMMON_TOKEN token);
 static  void			setText					(pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);
 static  void			setText8				(pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);
 static	ANTLR3_UINT32   getType					(pANTLR3_COMMON_TOKEN token);
 static  void			setType					(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);
 static  ANTLR3_UINT32   getLine					(pANTLR3_COMMON_TOKEN token);
 static  void			setLine					(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);
 static  ANTLR3_INT32    getCharPositionInLine	(pANTLR3_COMMON_TOKEN token);
 static  void			setCharPositionInLine	(pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);
 static  ANTLR3_UINT32   getChannel				(pANTLR3_COMMON_TOKEN token);
 static  void			setChannel				(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);
 static  ANTLR3_MARKER   getTokenIndex			(pANTLR3_COMMON_TOKEN token);
 static  void			setTokenIndex			(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);
 static  ANTLR3_MARKER   getStartIndex			(pANTLR3_COMMON_TOKEN token);
 static  void			setStartIndex			(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
 static  ANTLR3_MARKER   getStopIndex			(pANTLR3_COMMON_TOKEN token);
 static  void			setStopIndex			(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
 static  pANTLR3_STRING  toString				(pANTLR3_COMMON_TOKEN token);

 /* Factory API
  */
 static	void			factoryClose	(pANTLR3_TOKEN_FACTORY factory);
 static	pANTLR3_COMMON_TOKEN	newToken	(void);
 static  void			setInputStream	(pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input);
 static	void                    factoryReset    (pANTLR3_TOKEN_FACTORY factory);

 /* Internal management functions
  */
 static	void			newPool		(pANTLR3_TOKEN_FACTORY factory);
 static	pANTLR3_COMMON_TOKEN    newPoolToken	(pANTLR3_TOKEN_FACTORY factory);


 ANTLR3_API pANTLR3_COMMON_TOKEN
 antlr3CommonTokenNew(ANTLR3_UINT32 ttype)
 {
 	pANTLR3_COMMON_TOKEN    token;

 	// Create a raw token with the interface installed
 	//
 	token   = newToken();

 	if	(token != NULL)
 	{
 		token->setType(token, ttype);
 	}

 	// All good
 	//
 	return  token;
 }

 ANTLR3_API pANTLR3_TOKEN_FACTORY
 antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)
 {
     pANTLR3_TOKEN_FACTORY   factory;

     /* allocate memory
      */
     factory	= (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));

     if	(factory == NULL)
     {
 	return	NULL;
     }

     /* Install factory API
      */
     factory->newToken	    = newPoolToken;
     factory->close	    = factoryClose;
     factory->setInputStream = setInputStream;
     factory->reset          = factoryReset;

     /* Allocate the initial pool
      */
     factory->thisPool	= -1;
     factory->pools      = NULL;
     factory->maxPool    = -1;
     newPool(factory);

     /* Factory space is good, we now want to initialize our cheating token
      * which one it is initialized is the model for all tokens we manufacture
      */
     antlr3SetTokenAPI(&factory->unTruc);

     /* Set some initial variables for future copying
      */
     factory->unTruc.factoryMade	= ANTLR3_TRUE;

     // Input stream
     //
     setInputStream(factory, input);

     return  factory;

 }

 static void
 setInputStream	(pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input)
 {
     factory->input          =  input;
     factory->unTruc.input   =  input;
 	if	(input != NULL)
 	{
 		factory->unTruc.strFactory	= input->strFactory;
 	}
 	else
 	{
 		factory->unTruc.strFactory = NULL;
     }
 }

 static void
 newPool(pANTLR3_TOKEN_FACTORY factory)
 {
     /* Increment factory count
      */
     factory->thisPool++;

     // If we were reusing this token factory then we may already have a pool
     // allocated. If we exceeded the max avaible then we must allocate a new
     // one.
     if  (factory->thisPool > factory->maxPool)
     {
         /* Ensure we have enough pointers allocated
          */
         factory->pools = (pANTLR3_COMMON_TOKEN *)
 		         ANTLR3_REALLOC(	(void *)factory->pools,	    /* Current pools pointer (starts at NULL)	*/
 					    (ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *))	/* Memory for new pool pointers */
 					    );

         /* Allocate a new pool for the factory
          */
         factory->pools[factory->thisPool]	=
 			        (pANTLR3_COMMON_TOKEN)
 				    ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));

         // We now have a new pool and can track it as the maximum we have created so far
         //
         factory->maxPool = factory->thisPool;
     }

     /* Reset the counters
      */
     factory->nextToken	= 0;

     /* Done
      */
     return;
 }

 static pANTLR3_COMMON_TOKEN
 newPoolToken(pANTLR3_TOKEN_FACTORY factory)
 {
     pANTLR3_COMMON_TOKEN token;

     /* See if we need a new token pool before allocating a new
      * one
      */
     if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE)
     {
         /* We ran out of tokens in the current pool, so we need a new pool
          */
         newPool(factory);
     }

     /* Assuming everything went well (we are trying for performance here so doing minimal
      * error checking. Then we can work out what the pointer is to the next token.
      */
     token = factory->pools[factory->thisPool] + factory->nextToken;
     factory->nextToken++;

     /* We have our token pointer now, so we can initialize it to the predefined model.
      * We only need do this though if the token is not already initialized, we just check
      * an api function pointer for this as they are allocated via calloc.
      */
     if  (token->setStartIndex == NULL)
     {
         antlr3SetTokenAPI(token);

         // It is factory made, and we need to copy the string factory pointer
         //
         token->factoryMade  = ANTLR3_TRUE;
         token->strFactory   = factory->input == NULL ? NULL : factory->input->strFactory;
         token->input        = factory->input;
     }

     /* And we are done
      */
     return token;
 }

 static	void
 factoryReset	    (pANTLR3_TOKEN_FACTORY factory)
 {
     // Just start again with pool #0 when we are
     // called.
     //
     factory->thisPool   = -1;
     newPool(factory);
 }

 static	void
 factoryClose	    (pANTLR3_TOKEN_FACTORY factory)
 {
     pANTLR3_COMMON_TOKEN    pool;
     ANTLR3_INT32	    poolCount;
     ANTLR3_UINT32	    limit;
     ANTLR3_UINT32	    token;
     pANTLR3_COMMON_TOKEN    check;

     /* We iterate the token pools one at a time
      */
     for	(poolCount = 0; poolCount <= factory->thisPool; poolCount++)
     {
 	/* Pointer to current pool
 	 */
 	pool	= factory->pools[poolCount];

 	/* Work out how many tokens we need to check in this pool.
 	 */
 	limit	= (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE);

 	/* Marginal condition, we might be at the start of a brand new pool
 	 * where the nextToken is 0 and nothing has been allocated.
 	 */
 	if  (limit > 0)
 	{
 	    /* We have some tokens allocated from this pool
 	     */
 	    for (token = 0; token < limit; token++)
 	    {
 		/* Next one in the chain
 		 */
 		check	= pool + token;

 		/* If the programmer made this a custom token, then
 		 * see if we need to call their free routine.
 		 */
 		if  (check->custom != NULL && check->freeCustom != NULL)
 		{
 		    check->freeCustom(check->custom);
 		    check->custom = NULL;
 		}
 	    }
 	}

 	/* We can now free this pool allocation
 	 */
 	ANTLR3_FREE(factory->pools[poolCount]);
 	factory->pools[poolCount] = NULL;
     }

     /* All the pools are deallocated we can free the pointers to the pools
      * now.
      */
     ANTLR3_FREE(factory->pools);

     /* Finally, we can free the space for the factory itself
      */
     ANTLR3_FREE(factory);
 }


 static	pANTLR3_COMMON_TOKEN
 newToken(void)
 {
     pANTLR3_COMMON_TOKEN    token;

     /* Allocate memory for this
      */
     token   = (pANTLR3_COMMON_TOKEN) ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN)));

     if	(token == NULL)
     {
 	return	NULL;
     }

     // Install the API
     //
     antlr3SetTokenAPI(token);
     token->factoryMade = ANTLR3_FALSE;

     return  token;
 }

 ANTLR3_API void
 antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)
 {
     token->getText		    = getText;
     token->setText		    = setText;
     token->setText8		    = setText8;
     token->getType		    = getType;
     token->setType		    = setType;
     token->getLine		    = getLine;
     token->setLine		    = setLine;
     token->setLine		    = setLine;
     token->getCharPositionInLine    = getCharPositionInLine;
     token->setCharPositionInLine    = setCharPositionInLine;
     token->getChannel		    = getChannel;
     token->setChannel		    = setChannel;
     token->getTokenIndex	    = getTokenIndex;
     token->setTokenIndex	    = setTokenIndex;
     token->getStartIndex	    = getStartIndex;
     token->setStartIndex	    = setStartIndex;
     token->getStopIndex		    = getStopIndex;
     token->setStopIndex		    = setStopIndex;
     token->toString		    = toString;

     return;
 }

 static  pANTLR3_STRING  getText			(pANTLR3_COMMON_TOKEN token)
 {
 	switch (token->textState)
 	{
 		case ANTLR3_TEXT_STRING:

 			// Someone already created a string for this token, so we just
 			// use it.
 			//
 			return	token->tokText.text;
 			break;

 		case ANTLR3_TEXT_CHARP:

 			// We had a straight text pointer installed, now we
 			// must convert it to a string. Note we have to do this here
 			// or otherwise setText8() will just install the same char*
 			//
 			if	(token->strFactory != NULL)
 			{
 				token->tokText.text	= token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);
 				token->textState	= ANTLR3_TEXT_STRING;
 				return token->tokText.text;
 			}
 			else
 			{
 				// We cannot do anything here
 				//
 				return NULL;
 			}
 			break;

 		default:

 			// EOF is a special case
 			//
 			if (token->type == ANTLR3_TOKEN_EOF)
 			{
 				token->tokText.text				= token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");
 				token->textState				= ANTLR3_TEXT_STRING;
 				token->tokText.text->factory	= token->strFactory;
 				return token->tokText.text;
 			}


 			// We had nothing installed in the token, create a new string
 			// from the input stream
 			//

 			if	(token->input != NULL)
 			{

 				return	token->input->substr(	token->input,
 												token->getStartIndex(token),
  												token->getStopIndex(token)
 											);
 			}

 			// Nothing to return, there is no input stream
 			//
 			return NULL;
 			break;
 	}
 }
 static  void		setText8		(pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)
 {
 	// No text to set, so ignore
 	//
 	if	(text == NULL) return;

 	switch	(token->textState)
 	{
 		case	ANTLR3_TEXT_NONE:
 		case	ANTLR3_TEXT_CHARP:	// Caller must free before setting again, if it needs to be freed

 			// Nothing in there yet, or just a char *, so just set the
 			// text as a pointer
 			//
 			token->textState		= ANTLR3_TEXT_CHARP;
 			token->tokText.chars	= (pANTLR3_UCHAR)text;
 			break;

 		default:

 			// It was already a pANTLR3_STRING, so just override it
 			//
 			token->tokText.text->set8(token->tokText.text, (const char *)text);
 			break;
 	}

 	// We are done
 	//
 	return;
 }

 /** \brief Install the supplied text string as teh text for the token.
  * The method assumes that the existing text (if any) was created by a factory
  * and so does not attempt to release any memory it is using.Text not created
  * by a string fctory (not advised) should be released prior to this call.
  */
 static  void		setText			(pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)
 {
 	// Merely replaces and existing pre-defined text with the supplied
 	// string
 	//
 	token->textState	= ANTLR3_TEXT_STRING;
 	token->tokText.text	= text;

 	/* We are done
 	*/
 	return;
 }

 static	ANTLR3_UINT32   getType			(pANTLR3_COMMON_TOKEN token)
 {
     return  token->type;
 }

 static  void		setType			(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)
 {
     token->type = type;
 }

 static  ANTLR3_UINT32   getLine			(pANTLR3_COMMON_TOKEN token)
 {
     return  token->line;
 }

 static  void		setLine			(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)
 {
     token->line = line;
 }

 static  ANTLR3_INT32    getCharPositionInLine	(pANTLR3_COMMON_TOKEN token)
 {
     return  token->charPosition;
 }

 static  void		setCharPositionInLine	(pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)
 {
     token->charPosition = pos;
 }

 static  ANTLR3_UINT32   getChannel		(pANTLR3_COMMON_TOKEN token)
 {
     return  token->channel;
 }

 static  void		setChannel		(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)
 {
     token->channel  = channel;
 }

 static  ANTLR3_MARKER   getTokenIndex		(pANTLR3_COMMON_TOKEN token)
 {
     return  token->index;
 }

 static  void		setTokenIndex		(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)
 {
     token->index    = index;
 }

 static  ANTLR3_MARKER   getStartIndex		(pANTLR3_COMMON_TOKEN token)
 {
 	return  token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;
 }

 static  void		setStartIndex		(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)
 {
     token->start    = start;
 }

 static  ANTLR3_MARKER   getStopIndex		(pANTLR3_COMMON_TOKEN token)
 {
     return  token->stop;
 }

 static  void		setStopIndex		(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)
 {
     token->stop	= stop;
 }

 static  pANTLR3_STRING    toString		(pANTLR3_COMMON_TOKEN token)
 {
     pANTLR3_STRING  text;
     pANTLR3_STRING  outtext;

     text    =	token->getText(token);

     if	(text == NULL)
     {
 		return NULL;
     }

 	if	(text->factory == NULL)
 	{
 		return text;		// This usally means it is the EOF token
 	}

     /* A new empty string to assemble all the stuff in
      */
     outtext = text->factory->newRaw(text->factory);

     /* Now we use our handy dandy string utility to assemble the
      * the reporting string
      * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
      */
     outtext->append8(outtext, "[Index: ");
     outtext->addi   (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
     outtext->append8(outtext, " (Start: ");
     outtext->addi   (outtext, (ANTLR3_INT32)token->getStartIndex(token));
     outtext->append8(outtext, "-Stop: ");
     outtext->addi   (outtext, (ANTLR3_INT32)token->getStopIndex(token));
     outtext->append8(outtext, ") ='");
     outtext->appendS(outtext, text);
     outtext->append8(outtext, "', type<");
     outtext->addi   (outtext, token->type);
     outtext->append8(outtext, "> ");

     if	(token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
     {
 		outtext->append8(outtext, "(channel = ");
 		outtext->addi	(outtext, (ANTLR3_INT32)token->getChannel(token));
 		outtext->append8(outtext, ") ");
     }

     outtext->append8(outtext, "Line: ");
     outtext->addi   (outtext, (ANTLR3_INT32)token->getLine(token));
     outtext->append8(outtext, " LinePos:");
     outtext->addi   (outtext, token->getCharPositionInLine(token));
     outtext->addc   (outtext, ']');

     return  outtext;
 }
	/**
	* Contains the default implementation of the common token used within
	* java. Custom tokens should create this structure and then append to it using the
	* custom pointer to install their own structure and API.
	*/

	// [The "BSD licence"]
	// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
	// http://www.temporal-wave.com
	// http://www.linkedin.com/in/jimidle
	//
	// All rights reserved.
	//
	// Redistribution and use in source and binary forms, with or without
	// modification, are permitted provided that the following conditions
	// are met:
	// 1. Redistributions of source code must retain the above copyright
	// notice, this list of conditions and the following disclaimer.
	// 2. Redistributions in binary form must reproduce the above copyright
	// notice, this list of conditions and the following disclaimer in the
	// documentation and/or other materials provided with the distribution.
	// 3. The name of the author may not be used to endorse or promote products
	// derived from this software without specific prior written permission.
	//
	// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	#include <antlr3.h>

	/* Token API
	*/
	static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token);
	static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);
	static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);
	static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token);
	static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);
	static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token);
	static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);
	static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token);
	static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);
	static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token);
	static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);
	static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token);
	static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);
	static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token);
	static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
	static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token);
	static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
	static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token);

	/* Factory API
	*/
	static void factoryClose (pANTLR3_TOKEN_FACTORY factory);
	static pANTLR3_COMMON_TOKEN newToken (void);
	static void setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input);
	static void factoryReset (pANTLR3_TOKEN_FACTORY factory);

	/* Internal management functions
	*/
	static void newPool (pANTLR3_TOKEN_FACTORY factory);
	static pANTLR3_COMMON_TOKEN newPoolToken (pANTLR3_TOKEN_FACTORY factory);


	ANTLR3_API pANTLR3_COMMON_TOKEN
	antlr3CommonTokenNew(ANTLR3_UINT32 ttype)
	{
	pANTLR3_COMMON_TOKEN token;

	// Create a raw token with the interface installed
	//
	token = newToken();

	if (token != NULL)
	{
	token->setType(token, ttype);
	}

	// All good
	//
	return token;
	}

	ANTLR3_API pANTLR3_TOKEN_FACTORY
	antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)
	{
	pANTLR3_TOKEN_FACTORY factory;

	/* allocate memory
	*/
	factory = (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));

	if (factory == NULL)
	{
	return NULL;
	}

	/* Install factory API
	*/
	factory->newToken = newPoolToken;
	factory->close = factoryClose;
	factory->setInputStream = setInputStream;
	factory->reset = factoryReset;

	/* Allocate the initial pool
	*/
	factory->thisPool = -1;
	factory->pools = NULL;
	factory->maxPool = -1;
	newPool(factory);

	/* Factory space is good, we now want to initialize our cheating token
	* which one it is initialized is the model for all tokens we manufacture
	*/
	antlr3SetTokenAPI(&factory->unTruc);

	/* Set some initial variables for future copying
	*/
	factory->unTruc.factoryMade = ANTLR3_TRUE;

	// Input stream
	//
	setInputStream(factory, input);

	return factory;

	}

	static void
	setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input)
	{
	factory->input = input;
	factory->unTruc.input = input;
	if (input != NULL)
	{
	factory->unTruc.strFactory = input->strFactory;
	}
	else
	{
	factory->unTruc.strFactory = NULL;
	}
	}

	static void
	newPool(pANTLR3_TOKEN_FACTORY factory)
	{
	/* Increment factory count
	*/
	factory->thisPool++;

	// If we were reusing this token factory then we may already have a pool
	// allocated. If we exceeded the max avaible then we must allocate a new
	// one.
	if (factory->thisPool > factory->maxPool)
	{
	/* Ensure we have enough pointers allocated
	*/
	factory->pools = (pANTLR3_COMMON_TOKEN *)
	ANTLR3_REALLOC( (void )factory->pools, / Current pools pointer (starts at NULL) */
	(ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN )) / Memory for new pool pointers */
	);

	/* Allocate a new pool for the factory
	*/
	factory->pools[factory->thisPool] =
	(pANTLR3_COMMON_TOKEN)
	ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));

	// We now have a new pool and can track it as the maximum we have created so far
	//
	factory->maxPool = factory->thisPool;
	}

	/* Reset the counters
	*/
	factory->nextToken = 0;

	/* Done
	*/
	return;
	}

	static pANTLR3_COMMON_TOKEN
	newPoolToken(pANTLR3_TOKEN_FACTORY factory)
	{
	pANTLR3_COMMON_TOKEN token;

	/* See if we need a new token pool before allocating a new
	* one
	*/
	if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE)
	{
	/* We ran out of tokens in the current pool, so we need a new pool
	*/
	newPool(factory);
	}

	/* Assuming everything went well (we are trying for performance here so doing minimal
	* error checking. Then we can work out what the pointer is to the next token.
	*/
	token = factory->pools[factory->thisPool] + factory->nextToken;
	factory->nextToken++;

	/* We have our token pointer now, so we can initialize it to the predefined model.
	* We only need do this though if the token is not already initialized, we just check
	* an api function pointer for this as they are allocated via calloc.
	*/
	if (token->setStartIndex == NULL)
	{
	antlr3SetTokenAPI(token);

	// It is factory made, and we need to copy the string factory pointer
	//
	token->factoryMade = ANTLR3_TRUE;
	token->strFactory = factory->input == NULL ? NULL : factory->input->strFactory;
	token->input = factory->input;
	}

	/* And we are done
	*/
	return token;
	}

	static void
	factoryReset (pANTLR3_TOKEN_FACTORY factory)
	{
	// Just start again with pool #0 when we are
	// called.
	//
	factory->thisPool = -1;
	newPool(factory);
	}

	static void
	factoryClose (pANTLR3_TOKEN_FACTORY factory)
	{
	pANTLR3_COMMON_TOKEN pool;
	ANTLR3_INT32 poolCount;
	ANTLR3_UINT32 limit;
	ANTLR3_UINT32 token;
	pANTLR3_COMMON_TOKEN check;

	/* We iterate the token pools one at a time
	*/
	for (poolCount = 0; poolCount <= factory->thisPool; poolCount++)
	{
	/* Pointer to current pool
	*/
	pool = factory->pools[poolCount];

	/* Work out how many tokens we need to check in this pool.
	*/
	limit = (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE);

	/* Marginal condition, we might be at the start of a brand new pool
	* where the nextToken is 0 and nothing has been allocated.
	*/
	if (limit > 0)
	{
	/* We have some tokens allocated from this pool
	*/
	for (token = 0; token < limit; token++)
	{
	/* Next one in the chain
	*/
	check = pool + token;

	/* If the programmer made this a custom token, then
	* see if we need to call their free routine.
	*/
	if (check->custom != NULL && check->freeCustom != NULL)
	{
	check->freeCustom(check->custom);
	check->custom = NULL;
	}
	}
	}

	/* We can now free this pool allocation
	*/
	ANTLR3_FREE(factory->pools[poolCount]);
	factory->pools[poolCount] = NULL;
	}

	/* All the pools are deallocated we can free the pointers to the pools
	* now.
	*/
	ANTLR3_FREE(factory->pools);

	/* Finally, we can free the space for the factory itself
	*/
	ANTLR3_FREE(factory);
	}


	static pANTLR3_COMMON_TOKEN
	newToken(void)
	{
	pANTLR3_COMMON_TOKEN token;

	/* Allocate memory for this
	*/
	token = (pANTLR3_COMMON_TOKEN) ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN)));

	if (token == NULL)
	{
	return NULL;
	}

	// Install the API
	//
	antlr3SetTokenAPI(token);
	token->factoryMade = ANTLR3_FALSE;

	return token;
	}

	ANTLR3_API void
	antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)
	{
	token->getText = getText;
	token->setText = setText;
	token->setText8 = setText8;
	token->getType = getType;
	token->setType = setType;
	token->getLine = getLine;
	token->setLine = setLine;
	token->setLine = setLine;
	token->getCharPositionInLine = getCharPositionInLine;
	token->setCharPositionInLine = setCharPositionInLine;
	token->getChannel = getChannel;
	token->setChannel = setChannel;
	token->getTokenIndex = getTokenIndex;
	token->setTokenIndex = setTokenIndex;
	token->getStartIndex = getStartIndex;
	token->setStartIndex = setStartIndex;
	token->getStopIndex = getStopIndex;
	token->setStopIndex = setStopIndex;
	token->toString = toString;

	return;
	}

	static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token)
	{
	switch (token->textState)
	{
	case ANTLR3_TEXT_STRING:

	// Someone already created a string for this token, so we just
	// use it.
	//
	return token->tokText.text;
	break;

	case ANTLR3_TEXT_CHARP:

	// We had a straight text pointer installed, now we
	// must convert it to a string. Note we have to do this here
	// or otherwise setText8() will just install the same char*
	//
	if (token->strFactory != NULL)
	{
	token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);
	token->textState = ANTLR3_TEXT_STRING;
	return token->tokText.text;
	}
	else
	{
	// We cannot do anything here
	//
	return NULL;
	}
	break;

	default:

	// EOF is a special case
	//
	if (token->type == ANTLR3_TOKEN_EOF)
	{
	token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");
	token->textState = ANTLR3_TEXT_STRING;
	token->tokText.text->factory = token->strFactory;
	return token->tokText.text;
	}


	// We had nothing installed in the token, create a new string
	// from the input stream
	//

	if (token->input != NULL)
	{

	return token->input->substr( token->input,
	token->getStartIndex(token),
	token->getStopIndex(token)
	);
	}

	// Nothing to return, there is no input stream
	//
	return NULL;
	break;
	}
	}
	static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)
	{
	// No text to set, so ignore
	//
	if (text == NULL) return;

	switch (token->textState)
	{
	case ANTLR3_TEXT_NONE:
	case ANTLR3_TEXT_CHARP: // Caller must free before setting again, if it needs to be freed

	// Nothing in there yet, or just a char *, so just set the
	// text as a pointer
	//
	token->textState = ANTLR3_TEXT_CHARP;
	token->tokText.chars = (pANTLR3_UCHAR)text;
	break;

	default:

	// It was already a pANTLR3_STRING, so just override it
	//
	token->tokText.text->set8(token->tokText.text, (const char *)text);
	break;
	}

	// We are done
	//
	return;
	}

	/** \brief Install the supplied text string as teh text for the token.
	* The method assumes that the existing text (if any) was created by a factory
	* and so does not attempt to release any memory it is using.Text not created
	* by a string fctory (not advised) should be released prior to this call.
	*/
	static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)
	{
	// Merely replaces and existing pre-defined text with the supplied
	// string
	//
	token->textState = ANTLR3_TEXT_STRING;
	token->tokText.text = text;

	/* We are done
	*/
	return;
	}

	static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token)
	{
	return token->type;
	}

	static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)
	{
	token->type = type;
	}

	static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token)
	{
	return token->line;
	}

	static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)
	{
	token->line = line;
	}

	static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token)
	{
	return token->charPosition;
	}

	static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)
	{
	token->charPosition = pos;
	}

	static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token)
	{
	return token->channel;
	}

	static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)
	{
	token->channel = channel;
	}

	static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token)
	{
	return token->index;
	}

	static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)
	{
	token->index = index;
	}

	static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token)
	{
	return token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;
	}

	static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)
	{
	token->start = start;
	}

	static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token)
	{
	return token->stop;
	}

	static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)
	{
	token->stop = stop;
	}

	static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token)
	{
	pANTLR3_STRING text;
	pANTLR3_STRING outtext;

	text = token->getText(token);

	if (text == NULL)
	{
	return NULL;
	}

	if (text->factory == NULL)
	{
	return text; // This usally means it is the EOF token
	}

	/* A new empty string to assemble all the stuff in
	*/
	outtext = text->factory->newRaw(text->factory);

	/* Now we use our handy dandy string utility to assemble the
	* the reporting string
	* return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
	*/
	outtext->append8(outtext, "[Index: ");
	outtext->addi (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
	outtext->append8(outtext, " (Start: ");
	outtext->addi (outtext, (ANTLR3_INT32)token->getStartIndex(token));
	outtext->append8(outtext, "-Stop: ");
	outtext->addi (outtext, (ANTLR3_INT32)token->getStopIndex(token));
	outtext->append8(outtext, ") ='");
	outtext->appendS(outtext, text);
	outtext->append8(outtext, "', type<");
	outtext->addi (outtext, token->type);
	outtext->append8(outtext, "> ");

	if (token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
	{
	outtext->append8(outtext, "(channel = ");
	outtext->addi (outtext, (ANTLR3_INT32)token->getChannel(token));
	outtext->append8(outtext, ") ");
	}

	outtext->append8(outtext, "Line: ");
	outtext->addi (outtext, (ANTLR3_INT32)token->getLine(token));
	outtext->append8(outtext, " LinePos:");
	outtext->addi (outtext, token->getCharPositionInLine(token));
	outtext->addc (outtext, ']');

	return outtext;
	}