| /** \file |
| * \brief Defines the interface for a common token. |
| * |
| * All token streams should provide their tokens using an instance |
| * of this common token. A custom pointer is provided, wher you may attach |
| * a further structure to enhance the common token if you feel the need |
| * to do so. The C runtime will assume that a token provides implementations |
| * of the interface functions, but all of them may be rplaced by your own |
| * implementation if you require it. |
| */ |
| #ifndef _ANTLR3_COMMON_TOKEN_H |
| #define _ANTLR3_COMMON_TOKEN_H |
| |
| // [The "BSD licence"] |
| // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC |
| // http://www.temporal-wave.com |
| // http://www.linkedin.com/in/jimidle |
| // |
| // All rights reserved. |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions |
| // are met: |
| // 1. Redistributions of source code must retain the above copyright |
| // notice, this list of conditions and the following disclaimer. |
| // 2. Redistributions in binary form must reproduce the above copyright |
| // notice, this list of conditions and the following disclaimer in the |
| // documentation and/or other materials provided with the distribution. |
| // 3. The name of the author may not be used to endorse or promote products |
| // derived from this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
| // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
| // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
| // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| #include <antlr3defs.h> |
| |
| /** How many tokens to allocate at once in the token factory |
| */ |
| #define ANTLR3_FACTORY_POOL_SIZE 1024 |
| |
| /* Base token types, which all lexer/parser tokens come after in sequence. |
| */ |
| |
| /** Indicator of an invalid token |
| */ |
| #define ANTLR3_TOKEN_INVALID 0 |
| |
| #define ANTLR3_EOR_TOKEN_TYPE 1 |
| |
| /** Imaginary token type to cause a traversal of child nodes in a tree parser |
| */ |
| #define ANTLR3_TOKEN_DOWN 2 |
| |
| /** Imaginary token type to signal the end of a stream of child nodes. |
| */ |
| #define ANTLR3_TOKEN_UP 3 |
| |
| /** First token that can be used by users/generated code |
| */ |
| |
| #define ANTLR3_MIN_TOKEN_TYPE ANTLR3_TOKEN_UP + 1 |
| |
| /** End of file token |
| */ |
| #define ANTLR3_TOKEN_EOF (ANTLR3_CHARSTREAM_EOF & 0xFFFFFFFF) |
| |
| /** Default channel for a token |
| */ |
| #define ANTLR3_TOKEN_DEFAULT_CHANNEL 0 |
| |
| /** Reserved channel number for a HIDDEN token - a token that |
| * is hidden from the parser. |
| */ |
| #define HIDDEN 99 |
| |
| #ifdef __cplusplus |
| extern "C" { |
| #endif |
| |
| // Indicates whether this token is carrying: |
| // |
| // State | Meaning |
| // ------+-------------------------------------- |
| // 0 | Nothing (neither rewrite text, nor setText) |
| // 1 | char * to user supplied rewrite text |
| // 2 | pANTLR3_STRING because of setText or similar action |
| // |
| #define ANTLR3_TEXT_NONE 0 |
| #define ANTLR3_TEXT_CHARP 1 |
| #define ANTLR3_TEXT_STRING 2 |
| |
| /** The definition of an ANTLR3 common token structure, which all implementations |
| * of a token stream should provide, installing any further structures in the |
| * custom pointer element of this structure. |
| * |
| * \remark |
| * Token streams are in essence provided by lexers or other programs that serve |
| * as lexers. |
| */ |
| typedef struct ANTLR3_COMMON_TOKEN_struct |
| { |
| /** The actual type of this token |
| */ |
| ANTLR3_UINT32 type; |
| |
| /** Indicates that a token was produced from the token factory and therefore |
| * the the freeToken() method should not do anything itself because |
| * token factory is responsible for deleting it. |
| */ |
| ANTLR3_BOOLEAN factoryMade; |
| |
| /// A string factory that we can use if we ever need the text of a token |
| /// and need to manufacture a pANTLR3_STRING |
| /// |
| pANTLR3_STRING_FACTORY strFactory; |
| |
| /** The line number in the input stream where this token was derived from |
| */ |
| ANTLR3_UINT32 line; |
| |
| /** The offset into the input stream that the line in which this |
| * token resides starts. |
| */ |
| void * lineStart; |
| |
| /** The character position in the line that this token was derived from |
| */ |
| ANTLR3_INT32 charPosition; |
| |
| /** The virtual channel that this token exists in. |
| */ |
| ANTLR3_UINT32 channel; |
| |
| /** Pointer to the input stream that this token originated in. |
| */ |
| pANTLR3_INPUT_STREAM input; |
| |
| /** What the index of this token is, 0, 1, .., n-2, n-1 tokens |
| */ |
| ANTLR3_MARKER index; |
| |
| /** The character offset in the input stream where the text for this token |
| * starts. |
| */ |
| ANTLR3_MARKER start; |
| |
| /** The character offset in the input stream where the text for this token |
| * stops. |
| */ |
| ANTLR3_MARKER stop; |
| |
| /// Indicates whether this token is carrying: |
| /// |
| /// State | Meaning |
| /// ------+-------------------------------------- |
| /// 0 | Nothing (neither rewrite text, nor setText) |
| /// 1 | char * to user supplied rewrite text |
| /// 2 | pANTLR3_STRING because of setText or similar action |
| /// |
| /// Affects the union structure tokText below |
| /// (uses 32 bit so alignment is always good) |
| /// |
| ANTLR3_UINT32 textState; |
| |
| union |
| { |
| /// Pointer that is used when the token just has a pointer to |
| /// a char *, such as when a rewrite of an imaginary token supplies |
| /// a string in the grammar. No sense in constructing a pANTLR3_STRING just |
| /// for that, as mostly the text will not be accessed - if it is, then |
| /// we will build a pANTLR3_STRING for it a that point. |
| /// |
| pANTLR3_UCHAR chars; |
| |
| /// Some token types actually do carry around their associated text, hence |
| /// (*getText)() will return this pointer if it is not NULL |
| /// |
| pANTLR3_STRING text; |
| } |
| tokText; |
| |
| /** Because it is a bit more of a hassle to override an ANTLR3_COMMON_TOKEN |
| * as the standard structure for a token, a number of user programmable |
| * elements are allowed in a token. This is one of them. |
| */ |
| ANTLR3_UINT32 user1; |
| |
| /** Because it is a bit more of a hassle to override an ANTLR3_COMMON_TOKEN |
| * as the standard structure for a token, a number of user programmable |
| * elements are allowed in a token. This is one of them. |
| */ |
| ANTLR3_UINT32 user2; |
| |
| /** Because it is a bit more of a hassle to override an ANTLR3_COMMON_TOKEN |
| * as the standard structure for a token, a number of user programmable |
| * elements are allowed in a token. This is one of them. |
| */ |
| ANTLR3_UINT32 user3; |
| |
| /** Pointer to a custom element that the ANTLR3 programmer may define and install |
| */ |
| void * custom; |
| |
| /** Pointer to a function that knows how to free the custom structure when the |
| * token is destroyed. |
| */ |
| void (*freeCustom)(void * custom); |
| |
| /* ============================== |
| * API |
| */ |
| |
| /** Pointer to function that returns the text pointer of a token, use |
| * toString() if you want a pANTLR3_STRING version of the token. |
| */ |
| pANTLR3_STRING (*getText)(struct ANTLR3_COMMON_TOKEN_struct * token); |
| |
| /** Pointer to a function that 'might' be able to set the text associated |
| * with a token. Imaginary tokens such as an ANTLR3_CLASSIC_TOKEN may actually |
| * do this, however many tokens such as ANTLR3_COMMON_TOKEN do not actaully have |
| * strings associated with them but just point into the current input stream. These |
| * tokens will implement this function with a function that errors out (probably |
| * drastically. |
| */ |
| void (*setText)(struct ANTLR3_COMMON_TOKEN_struct * token, pANTLR3_STRING text); |
| |
| /** Pointer to a function that 'might' be able to set the text associated |
| * with a token. Imaginary tokens such as an ANTLR3_CLASSIC_TOKEN may actually |
| * do this, however many tokens such as ANTLR3_COMMON_TOKEN do not actully have |
| * strings associated with them but just point into the current input stream. These |
| * tokens will implement this function with a function that errors out (probably |
| * drastically. |
| */ |
| void (*setText8)(struct ANTLR3_COMMON_TOKEN_struct * token, pANTLR3_UINT8 text); |
| |
| /** Pointer to a function that returns the token type of this token |
| */ |
| ANTLR3_UINT32 (*getType)(struct ANTLR3_COMMON_TOKEN_struct * token); |
| |
| /** Pointer to a function that sets the type of this token |
| */ |
| void (*setType)(struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_UINT32 ttype); |
| |
| /** Pointer to a function that gets the 'line' number where this token resides |
| */ |
| ANTLR3_UINT32 (*getLine)(struct ANTLR3_COMMON_TOKEN_struct * token); |
| |
| /** Pointer to a function that sets the 'line' number where this token reside |
| */ |
| void (*setLine)(struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_UINT32 line); |
| |
| /** Pointer to a function that gets the offset in the line where this token exists |
| */ |
| ANTLR3_INT32 (*getCharPositionInLine) (struct ANTLR3_COMMON_TOKEN_struct * token); |
| |
| /** Pointer to a function that sets the offset in the line where this token exists |
| */ |
| void (*setCharPositionInLine) (struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_INT32 pos); |
| |
| /** Pointer to a function that gets the channel that this token was placed in (parsers |
| * can 'tune' to these channels. |
| */ |
| ANTLR3_UINT32 (*getChannel) (struct ANTLR3_COMMON_TOKEN_struct * token); |
| |
| /** Pointer to a function that sets the channel that this token should belong to |
| */ |
| void (*setChannel) (struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_UINT32 channel); |
| |
| /** Pointer to a function that returns an index 0...n-1 of the token in the token |
| * input stream. |
| */ |
| ANTLR3_MARKER (*getTokenIndex) (struct ANTLR3_COMMON_TOKEN_struct * token); |
| |
| /** Pointer to a function that can set the token index of this token in the token |
| * input stream. |
| */ |
| void (*setTokenIndex) (struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_MARKER); |
| |
| /** Pointer to a function that gets the start index in the input stream for this token. |
| */ |
| ANTLR3_MARKER (*getStartIndex) (struct ANTLR3_COMMON_TOKEN_struct * token); |
| |
| /** Pointer to a function that sets the start index in the input stream for this token. |
| */ |
| void (*setStartIndex) (struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_MARKER index); |
| |
| /** Pointer to a function that gets the stop index in the input stream for this token. |
| */ |
| ANTLR3_MARKER (*getStopIndex) (struct ANTLR3_COMMON_TOKEN_struct * token); |
| |
| /** Pointer to a function that sets the stop index in the input stream for this token. |
| */ |
| void (*setStopIndex) (struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_MARKER index); |
| |
| /** Pointer to a function that returns this token as a text representation that can be |
| * printed with embedded control codes such as \n replaced with the printable sequence "\\n" |
| * This also yields a string structure that can be used more easily than the pointer to |
| * the input stream in certain situations. |
| */ |
| pANTLR3_STRING (*toString) (struct ANTLR3_COMMON_TOKEN_struct * token); |
| } |
| ANTLR3_COMMON_TOKEN; |
| |
| /** \brief ANTLR3 Token factory interface to create lots of tokens efficiently |
| * rather than creating and freeing lots of little bits of memory. |
| */ |
| typedef struct ANTLR3_TOKEN_FACTORY_struct |
| { |
| /** Pointers to the array of tokens that this factory has produced so far |
| */ |
| pANTLR3_COMMON_TOKEN *pools; |
| |
| /** Current pool tokens we are allocating from |
| */ |
| ANTLR3_INT32 thisPool; |
| |
| /** Maximum pool count we have available |
| */ |
| ANTLR3_INT32 maxPool; |
| |
| /** The next token to throw out from the pool, will cause a new pool allocation |
| * if this exceeds the available tokenCount |
| */ |
| ANTLR3_UINT32 nextToken; |
| |
| /** Trick to initialize tokens and their API quickly, we set up this token when the |
| * factory is created, then just copy the memory it uses into the new token. |
| */ |
| ANTLR3_COMMON_TOKEN unTruc; |
| |
| /** Pointer to an input stream that is using this token factory (may be NULL) |
| * which will be assigned to the tokens automatically. |
| */ |
| pANTLR3_INPUT_STREAM input; |
| |
| /** Pointer to a function that returns a new token |
| */ |
| pANTLR3_COMMON_TOKEN (*newToken) (struct ANTLR3_TOKEN_FACTORY_struct * factory); |
| |
| /** Pointer to a function that resets the factory so you can reuse the pools it |
| * has laready allocated |
| */ |
| void (*reset) (struct ANTLR3_TOKEN_FACTORY_struct * factory); |
| |
| /** Pointer to a function that changes teh curent inptu stream so that |
| * new tokens are created with reference to their originating text. |
| */ |
| void (*setInputStream) (struct ANTLR3_TOKEN_FACTORY_struct * factory, pANTLR3_INPUT_STREAM input); |
| /** Pointer to a function the destroys the factory |
| */ |
| void (*close) (struct ANTLR3_TOKEN_FACTORY_struct * factory); |
| } |
| ANTLR3_TOKEN_FACTORY; |
| |
| #ifdef __cplusplus |
| } |
| #endif |
| |
| #endif |
