| /**************************************************************************** |
| ** |
| ** Copyright (C) 2001-2004 Roberto Raggi |
| ** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). |
| ** All rights reserved. |
| ** Contact: Nokia Corporation (qt-info@nokia.com) |
| ** |
| ** This file is part of the qt3to4 porting application of the Qt Toolkit. |
| ** |
| ** $QT_BEGIN_LICENSE:LGPL$ |
| ** GNU Lesser General Public License Usage |
| ** This file may be used under the terms of the GNU Lesser General Public |
| ** License version 2.1 as published by the Free Software Foundation and |
| ** appearing in the file LICENSE.LGPL included in the packaging of this |
| ** file. Please review the following information to ensure the GNU Lesser |
| ** General Public License version 2.1 requirements will be met: |
| ** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. |
| ** |
| ** In addition, as a special exception, Nokia gives you certain additional |
| ** rights. These rights are described in the Nokia Qt LGPL Exception |
| ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. |
| ** |
| ** GNU General Public License Usage |
| ** Alternatively, this file may be used under the terms of the GNU General |
| ** Public License version 3.0 as published by the Free Software Foundation |
| ** and appearing in the file LICENSE.GPL included in the packaging of this |
| ** file. Please review the following information to ensure the GNU General |
| ** Public License version 3.0 requirements will be met: |
| ** http://www.gnu.org/copyleft/gpl.html. |
| ** |
| ** Other Usage |
| ** Alternatively, this file may be used in accordance with the terms and |
| ** conditions contained in a signed written agreement between you and Nokia. |
| ** |
| ** |
| ** |
| ** |
| ** |
| ** $QT_END_LICENSE$ |
| ** |
| ****************************************************************************/ |
| |
| #ifndef TOKENIZER_H |
| #define TOKENIZER_H |
| |
| #include "tokenengine.h" |
| #include <QVector> |
| #include <QByteArray> |
| |
| QT_BEGIN_NAMESPACE |
| |
| class Tokenizer |
| { |
| public: |
| Tokenizer(); |
| ~Tokenizer(); |
| typedef void (Tokenizer::*scan_fun_ptr)(int *kind); |
| QVector<TokenEngine::Token> tokenize(QByteArray text); |
| private: |
| bool nextToken(TokenEngine::Token &tok); |
| |
| void scanChar(int *kind); |
| void scanUnicodeChar(int *kind); |
| void scanNewline(int *kind); |
| void scanWhiteSpaces(int *kind); |
| void scanCharLiteral(int *kind); |
| void scanStringLiteral(int *kind); |
| void scanNumberLiteral(int *kind); |
| void scanIdentifier(int *kind); |
| void scanComment(int *kind); |
| void scanPreprocessor(int *kind); |
| void scanOperator(int *kind); |
| |
| void setupScanTable(); |
| private: |
| static bool s_initialized; |
| static scan_fun_ptr s_scan_table[]; |
| static int s_attr_table[]; |
| |
| const char *m_buffer; |
| int m_ptr; |
| |
| QVector<TokenEngine::Token> m_tokens; |
| }; |
| |
| QT_END_NAMESPACE |
| |
| #endif |