| /** |
| * Contains the default implementation of the common token used within |
| * java. Custom tokens should create this structure and then append to it using the |
| * custom pointer to install their own structure and API. |
| */ |
| |
| // [The "BSD licence"] |
| // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC |
| // http://www.temporal-wave.com |
| // http://www.linkedin.com/in/jimidle |
| // |
| // All rights reserved. |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions |
| // are met: |
| // 1. Redistributions of source code must retain the above copyright |
| // notice, this list of conditions and the following disclaimer. |
| // 2. Redistributions in binary form must reproduce the above copyright |
| // notice, this list of conditions and the following disclaimer in the |
| // documentation and/or other materials provided with the distribution. |
| // 3. The name of the author may not be used to endorse or promote products |
| // derived from this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
| // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
| // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
| // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| #include <antlr3.h> |
| |
| /* Token API |
| */ |
| static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token); |
| static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text); |
| static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text); |
| static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token); |
| static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type); |
| static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token); |
| static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line); |
| static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token); |
| static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos); |
| static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token); |
| static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel); |
| static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token); |
| static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER); |
| static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token); |
| static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index); |
| static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token); |
| static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index); |
| static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token); |
| |
| /* Factory API |
| */ |
| static void factoryClose (pANTLR3_TOKEN_FACTORY factory); |
| static pANTLR3_COMMON_TOKEN newToken (void); |
| static void setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input); |
| static void factoryReset (pANTLR3_TOKEN_FACTORY factory); |
| |
| /* Internal management functions |
| */ |
| static void newPool (pANTLR3_TOKEN_FACTORY factory); |
| static pANTLR3_COMMON_TOKEN newPoolToken (pANTLR3_TOKEN_FACTORY factory); |
| |
| |
| ANTLR3_API pANTLR3_COMMON_TOKEN |
| antlr3CommonTokenNew(ANTLR3_UINT32 ttype) |
| { |
| pANTLR3_COMMON_TOKEN token; |
| |
| // Create a raw token with the interface installed |
| // |
| token = newToken(); |
| |
| if (token != NULL) |
| { |
| token->setType(token, ttype); |
| } |
| |
| // All good |
| // |
| return token; |
| } |
| |
| ANTLR3_API pANTLR3_TOKEN_FACTORY |
| antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input) |
| { |
| pANTLR3_TOKEN_FACTORY factory; |
| |
| /* allocate memory |
| */ |
| factory = (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY)); |
| |
| if (factory == NULL) |
| { |
| return NULL; |
| } |
| |
| /* Install factory API |
| */ |
| factory->newToken = newPoolToken; |
| factory->close = factoryClose; |
| factory->setInputStream = setInputStream; |
| factory->reset = factoryReset; |
| |
| /* Allocate the initial pool |
| */ |
| factory->thisPool = -1; |
| factory->pools = NULL; |
| factory->maxPool = -1; |
| newPool(factory); |
| |
| /* Factory space is good, we now want to initialize our cheating token |
| * which one it is initialized is the model for all tokens we manufacture |
| */ |
| antlr3SetTokenAPI(&factory->unTruc); |
| |
| /* Set some initial variables for future copying |
| */ |
| factory->unTruc.factoryMade = ANTLR3_TRUE; |
| |
| // Input stream |
| // |
| setInputStream(factory, input); |
| |
| return factory; |
| |
| } |
| |
| static void |
| setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input) |
| { |
| factory->input = input; |
| factory->unTruc.input = input; |
| if (input != NULL) |
| { |
| factory->unTruc.strFactory = input->strFactory; |
| } |
| else |
| { |
| factory->unTruc.strFactory = NULL; |
| } |
| } |
| |
| static void |
| newPool(pANTLR3_TOKEN_FACTORY factory) |
| { |
| /* Increment factory count |
| */ |
| factory->thisPool++; |
| |
| // If we were reusing this token factory then we may already have a pool |
| // allocated. If we exceeded the max avaible then we must allocate a new |
| // one. |
| if (factory->thisPool > factory->maxPool) |
| { |
| /* Ensure we have enough pointers allocated |
| */ |
| factory->pools = (pANTLR3_COMMON_TOKEN *) |
| ANTLR3_REALLOC( (void *)factory->pools, /* Current pools pointer (starts at NULL) */ |
| (ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *)) /* Memory for new pool pointers */ |
| ); |
| |
| /* Allocate a new pool for the factory |
| */ |
| factory->pools[factory->thisPool] = |
| (pANTLR3_COMMON_TOKEN) |
| ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE)); |
| |
| // We now have a new pool and can track it as the maximum we have created so far |
| // |
| factory->maxPool = factory->thisPool; |
| } |
| |
| /* Reset the counters |
| */ |
| factory->nextToken = 0; |
| |
| /* Done |
| */ |
| return; |
| } |
| |
| static pANTLR3_COMMON_TOKEN |
| newPoolToken(pANTLR3_TOKEN_FACTORY factory) |
| { |
| pANTLR3_COMMON_TOKEN token; |
| |
| /* See if we need a new token pool before allocating a new |
| * one |
| */ |
| if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE) |
| { |
| /* We ran out of tokens in the current pool, so we need a new pool |
| */ |
| newPool(factory); |
| } |
| |
| /* Assuming everything went well (we are trying for performance here so doing minimal |
| * error checking. Then we can work out what the pointer is to the next token. |
| */ |
| token = factory->pools[factory->thisPool] + factory->nextToken; |
| factory->nextToken++; |
| |
| /* We have our token pointer now, so we can initialize it to the predefined model. |
| * We only need do this though if the token is not already initialized, we just check |
| * an api function pointer for this as they are allocated via calloc. |
| */ |
| if (token->setStartIndex == NULL) |
| { |
| antlr3SetTokenAPI(token); |
| |
| // It is factory made, and we need to copy the string factory pointer |
| // |
| token->factoryMade = ANTLR3_TRUE; |
| token->strFactory = factory->input == NULL ? NULL : factory->input->strFactory; |
| token->input = factory->input; |
| } |
| |
| /* And we are done |
| */ |
| return token; |
| } |
| |
| static void |
| factoryReset (pANTLR3_TOKEN_FACTORY factory) |
| { |
| // Just start again with pool #0 when we are |
| // called. |
| // |
| factory->thisPool = -1; |
| newPool(factory); |
| } |
| |
| static void |
| factoryClose (pANTLR3_TOKEN_FACTORY factory) |
| { |
| pANTLR3_COMMON_TOKEN pool; |
| ANTLR3_INT32 poolCount; |
| ANTLR3_UINT32 limit; |
| ANTLR3_UINT32 token; |
| pANTLR3_COMMON_TOKEN check; |
| |
| /* We iterate the token pools one at a time |
| */ |
| for (poolCount = 0; poolCount <= factory->thisPool; poolCount++) |
| { |
| /* Pointer to current pool |
| */ |
| pool = factory->pools[poolCount]; |
| |
| /* Work out how many tokens we need to check in this pool. |
| */ |
| limit = (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE); |
| |
| /* Marginal condition, we might be at the start of a brand new pool |
| * where the nextToken is 0 and nothing has been allocated. |
| */ |
| if (limit > 0) |
| { |
| /* We have some tokens allocated from this pool |
| */ |
| for (token = 0; token < limit; token++) |
| { |
| /* Next one in the chain |
| */ |
| check = pool + token; |
| |
| /* If the programmer made this a custom token, then |
| * see if we need to call their free routine. |
| */ |
| if (check->custom != NULL && check->freeCustom != NULL) |
| { |
| check->freeCustom(check->custom); |
| check->custom = NULL; |
| } |
| } |
| } |
| |
| /* We can now free this pool allocation |
| */ |
| ANTLR3_FREE(factory->pools[poolCount]); |
| factory->pools[poolCount] = NULL; |
| } |
| |
| /* All the pools are deallocated we can free the pointers to the pools |
| * now. |
| */ |
| ANTLR3_FREE(factory->pools); |
| |
| /* Finally, we can free the space for the factory itself |
| */ |
| ANTLR3_FREE(factory); |
| } |
| |
| |
| static pANTLR3_COMMON_TOKEN |
| newToken(void) |
| { |
| pANTLR3_COMMON_TOKEN token; |
| |
| /* Allocate memory for this |
| */ |
| token = (pANTLR3_COMMON_TOKEN) ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN))); |
| |
| if (token == NULL) |
| { |
| return NULL; |
| } |
| |
| // Install the API |
| // |
| antlr3SetTokenAPI(token); |
| token->factoryMade = ANTLR3_FALSE; |
| |
| return token; |
| } |
| |
| ANTLR3_API void |
| antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token) |
| { |
| token->getText = getText; |
| token->setText = setText; |
| token->setText8 = setText8; |
| token->getType = getType; |
| token->setType = setType; |
| token->getLine = getLine; |
| token->setLine = setLine; |
| token->setLine = setLine; |
| token->getCharPositionInLine = getCharPositionInLine; |
| token->setCharPositionInLine = setCharPositionInLine; |
| token->getChannel = getChannel; |
| token->setChannel = setChannel; |
| token->getTokenIndex = getTokenIndex; |
| token->setTokenIndex = setTokenIndex; |
| token->getStartIndex = getStartIndex; |
| token->setStartIndex = setStartIndex; |
| token->getStopIndex = getStopIndex; |
| token->setStopIndex = setStopIndex; |
| token->toString = toString; |
| |
| return; |
| } |
| |
| static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token) |
| { |
| switch (token->textState) |
| { |
| case ANTLR3_TEXT_STRING: |
| |
| // Someone already created a string for this token, so we just |
| // use it. |
| // |
| return token->tokText.text; |
| break; |
| |
| case ANTLR3_TEXT_CHARP: |
| |
| // We had a straight text pointer installed, now we |
| // must convert it to a string. Note we have to do this here |
| // or otherwise setText8() will just install the same char* |
| // |
| if (token->strFactory != NULL) |
| { |
| token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars); |
| token->textState = ANTLR3_TEXT_STRING; |
| return token->tokText.text; |
| } |
| else |
| { |
| // We cannot do anything here |
| // |
| return NULL; |
| } |
| break; |
| |
| default: |
| |
| // EOF is a special case |
| // |
| if (token->type == ANTLR3_TOKEN_EOF) |
| { |
| token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>"); |
| token->textState = ANTLR3_TEXT_STRING; |
| token->tokText.text->factory = token->strFactory; |
| return token->tokText.text; |
| } |
| |
| |
| // We had nothing installed in the token, create a new string |
| // from the input stream |
| // |
| |
| if (token->input != NULL) |
| { |
| |
| return token->input->substr( token->input, |
| token->getStartIndex(token), |
| token->getStopIndex(token) |
| ); |
| } |
| |
| // Nothing to return, there is no input stream |
| // |
| return NULL; |
| break; |
| } |
| } |
| static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text) |
| { |
| // No text to set, so ignore |
| // |
| if (text == NULL) return; |
| |
| switch (token->textState) |
| { |
| case ANTLR3_TEXT_NONE: |
| case ANTLR3_TEXT_CHARP: // Caller must free before setting again, if it needs to be freed |
| |
| // Nothing in there yet, or just a char *, so just set the |
| // text as a pointer |
| // |
| token->textState = ANTLR3_TEXT_CHARP; |
| token->tokText.chars = (pANTLR3_UCHAR)text; |
| break; |
| |
| default: |
| |
| // It was already a pANTLR3_STRING, so just override it |
| // |
| token->tokText.text->set8(token->tokText.text, (const char *)text); |
| break; |
| } |
| |
| // We are done |
| // |
| return; |
| } |
| |
| /** \brief Install the supplied text string as teh text for the token. |
| * The method assumes that the existing text (if any) was created by a factory |
| * and so does not attempt to release any memory it is using.Text not created |
| * by a string fctory (not advised) should be released prior to this call. |
| */ |
| static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text) |
| { |
| // Merely replaces and existing pre-defined text with the supplied |
| // string |
| // |
| token->textState = ANTLR3_TEXT_STRING; |
| token->tokText.text = text; |
| |
| /* We are done |
| */ |
| return; |
| } |
| |
| static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token) |
| { |
| return token->type; |
| } |
| |
| static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type) |
| { |
| token->type = type; |
| } |
| |
| static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token) |
| { |
| return token->line; |
| } |
| |
| static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line) |
| { |
| token->line = line; |
| } |
| |
| static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token) |
| { |
| return token->charPosition; |
| } |
| |
| static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos) |
| { |
| token->charPosition = pos; |
| } |
| |
| static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token) |
| { |
| return token->channel; |
| } |
| |
| static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel) |
| { |
| token->channel = channel; |
| } |
| |
| static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token) |
| { |
| return token->index; |
| } |
| |
| static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index) |
| { |
| token->index = index; |
| } |
| |
| static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token) |
| { |
| return token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start; |
| } |
| |
| static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start) |
| { |
| token->start = start; |
| } |
| |
| static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token) |
| { |
| return token->stop; |
| } |
| |
| static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop) |
| { |
| token->stop = stop; |
| } |
| |
| static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token) |
| { |
| pANTLR3_STRING text; |
| pANTLR3_STRING outtext; |
| |
| text = token->getText(token); |
| |
| if (text == NULL) |
| { |
| return NULL; |
| } |
| |
| if (text->factory == NULL) |
| { |
| return text; // This usally means it is the EOF token |
| } |
| |
| /* A new empty string to assemble all the stuff in |
| */ |
| outtext = text->factory->newRaw(text->factory); |
| |
| /* Now we use our handy dandy string utility to assemble the |
| * the reporting string |
| * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]"; |
| */ |
| outtext->append8(outtext, "[Index: "); |
| outtext->addi (outtext, (ANTLR3_INT32)token->getTokenIndex(token)); |
| outtext->append8(outtext, " (Start: "); |
| outtext->addi (outtext, (ANTLR3_INT32)token->getStartIndex(token)); |
| outtext->append8(outtext, "-Stop: "); |
| outtext->addi (outtext, (ANTLR3_INT32)token->getStopIndex(token)); |
| outtext->append8(outtext, ") ='"); |
| outtext->appendS(outtext, text); |
| outtext->append8(outtext, "', type<"); |
| outtext->addi (outtext, token->type); |
| outtext->append8(outtext, "> "); |
| |
| if (token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL) |
| { |
| outtext->append8(outtext, "(channel = "); |
| outtext->addi (outtext, (ANTLR3_INT32)token->getChannel(token)); |
| outtext->append8(outtext, ") "); |
| } |
| |
| outtext->append8(outtext, "Line: "); |
| outtext->addi (outtext, (ANTLR3_INT32)token->getLine(token)); |
| outtext->append8(outtext, " LinePos:"); |
| outtext->addi (outtext, token->getCharPositionInLine(token)); |
| outtext->addc (outtext, ']'); |
| |
| return outtext; |
| } |
| |