| /**************************************************************************** | |
| ** | |
| ** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). | |
| ** All rights reserved. | |
| ** Contact: Nokia Corporation (qt-info@nokia.com) | |
| ** | |
| ** This file is part of the QtXmlPatterns module of the Qt Toolkit. | |
| ** | |
| ** $QT_BEGIN_LICENSE:LGPL$ | |
| ** GNU Lesser General Public License Usage | |
| ** This file may be used under the terms of the GNU Lesser General Public | |
| ** License version 2.1 as published by the Free Software Foundation and | |
| ** appearing in the file LICENSE.LGPL included in the packaging of this | |
| ** file. Please review the following information to ensure the GNU Lesser | |
| ** General Public License version 2.1 requirements will be met: | |
| ** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. | |
| ** | |
| ** In addition, as a special exception, Nokia gives you certain additional | |
| ** rights. These rights are described in the Nokia Qt LGPL Exception | |
| ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. | |
| ** | |
| ** GNU General Public License Usage | |
| ** Alternatively, this file may be used under the terms of the GNU General | |
| ** Public License version 3.0 as published by the Free Software Foundation | |
| ** and appearing in the file LICENSE.GPL included in the packaging of this | |
| ** file. Please review the following information to ensure the GNU General | |
| ** Public License version 3.0 requirements will be met: | |
| ** http://www.gnu.org/copyleft/gpl.html. | |
| ** | |
| ** Other Usage | |
| ** Alternatively, this file may be used in accordance with the terms and | |
| ** conditions contained in a signed written agreement between you and Nokia. | |
| ** | |
| ** | |
| ** | |
| ** | |
| ** | |
| ** $QT_END_LICENSE$ | |
| ** | |
| ****************************************************************************/ | |
| #include <QByteArray> | |
| #include "qquerytransformparser_p.h" | |
| #include "qxquerytokenizer_p.h" | |
| #include "qtokenlookup.cpp" | |
| QT_BEGIN_NAMESPACE | |
| namespace QPatternist | |
| { | |
| #define handleWhitespace() \ | |
| { \ | |
| const TokenType t = consumeWhitespace(); \ | |
| if(t != SUCCESS) \ | |
| return Token(t); \ | |
| } | |
| XQueryTokenizer::XQueryTokenizer(const QString &query, | |
| const QUrl &location, | |
| const State startingState) : Tokenizer(location) | |
| , m_data(query) | |
| , m_length(query.length()) | |
| , m_state(startingState) | |
| , m_pos(0) | |
| , m_line(1) | |
| , m_columnOffset(0) | |
| , m_scanOnly(false) | |
| { | |
| Q_ASSERT(location.isValid() || location.isEmpty()); | |
| } | |
| const QChar XQueryTokenizer::current() const | |
| { | |
| if(m_pos < m_length) | |
| return m_data.at(m_pos); | |
| else | |
| return QChar(); | |
| } | |
| char XQueryTokenizer::peekCurrent() const | |
| { | |
| return current().toAscii(); | |
| } | |
| int XQueryTokenizer::peekForColonColon() const | |
| { | |
| /* Note, we don't modify m_pos in this function, so we need to do offset | |
| * calculations. */ | |
| int pos = m_pos; | |
| while(pos < m_length) | |
| { | |
| switch(m_data.at(pos).toAscii()) | |
| { | |
| /* Fallthrough these four. */ | |
| case ' ': | |
| case '\t': | |
| case '\n': | |
| case '\r': | |
| break; | |
| case ':': | |
| { | |
| if(peekAhead((pos - m_pos) + 1) == ':') | |
| return pos - m_pos; | |
| /* Fallthrough. */ | |
| } | |
| default: | |
| return -1; | |
| } | |
| ++pos; | |
| } | |
| return -1; | |
| } | |
| Tokenizer::Token XQueryTokenizer::tokenAndChangeState(const TokenType code, | |
| const State s, | |
| const int advance) | |
| { | |
| Q_ASSERT(advance >= 0); | |
| m_pos += advance; | |
| setState(s); | |
| return Token(code); | |
| } | |
| Tokenizer::Token XQueryTokenizer::tokenAndChangeState(const TokenType code, | |
| const QString &value, | |
| const State s) | |
| { | |
| setState(s); | |
| return Token(code, value); | |
| } | |
| Tokenizer::Token XQueryTokenizer::tokenAndAdvance(const TokenType code, | |
| const int advance) | |
| { | |
| Q_ASSERT(advance >= 0); | |
| m_pos += advance; | |
| return Token(code); | |
| } | |
| QString XQueryTokenizer::normalizeEOL(const QString &input, | |
| const CharacterSkips &characterSkips) | |
| { | |
| const int len = input.count(); | |
| QString result; | |
| /* The likely hood is rather high it'll be the same content. */ | |
| result.reserve(len); | |
| for(int i = 0; i < len; ++i) | |
| { | |
| const QChar &at = input.at(i); | |
| if(characterSkips.contains(i)) | |
| { | |
| result.append(at); | |
| continue; | |
| } | |
| switch(input.at(i).unicode()) | |
| { | |
| case '\r': | |
| { | |
| if(i + 1 < len && input.at(i + 1) == QLatin1Char('\n')) | |
| ++i; | |
| /* Else, fallthrough. */ | |
| } | |
| case '\n': | |
| { | |
| result.append(QLatin1Char('\n')); | |
| continue; | |
| } | |
| default: | |
| { | |
| result.append(at); | |
| } | |
| } | |
| } | |
| return result; | |
| } | |
| Tokenizer::TokenType XQueryTokenizer::consumeComment() | |
| { | |
| /* Below, we return ERROR instead of END_OF_FILE such that the parser | |
| * sees an invalid comment. */ | |
| while(m_pos < m_length) | |
| { | |
| switch(peekCurrent()) | |
| { | |
| case ':': | |
| { | |
| ++m_pos; /* Consume ':' */ | |
| if(atEnd()) | |
| return ERROR; | |
| if(peekCurrent() == ')') | |
| { | |
| ++m_pos; /* Consume ')' */ | |
| return SUCCESS; /* The comment closed nicely. */ | |
| } | |
| continue; /* We don't want to increment m_pos twice. */ | |
| } | |
| case '(': | |
| { /* It looks like the start of a comment. */ | |
| ++m_pos; | |
| if(atEnd()) | |
| return END_OF_FILE; | |
| else if(peekCurrent() == ':') | |
| { | |
| /* And it is a nested comment -- parse it. */ | |
| const TokenType retval = consumeComment(); | |
| if(retval == SUCCESS) | |
| continue; /* Continue with our "own" comment. */ | |
| else | |
| return retval; /* Return the error in the nested comment. */ | |
| } | |
| break; | |
| } | |
| case '\n': | |
| /* Fallthrough. */ | |
| case '\r': | |
| { | |
| /* We want to count \r\n as a single line break. */ | |
| if(peekAhead() == '\n') | |
| ++m_pos; | |
| m_columnOffset = m_pos; | |
| ++m_line; | |
| break; | |
| } | |
| } | |
| ++m_pos; | |
| } | |
| return ERROR; /* Error: we reached the end while inside a comment. */ | |
| } | |
| bool XQueryTokenizer::consumeRawWhitespace() | |
| { | |
| while(m_pos < m_length) | |
| { | |
| switch(peekCurrent()) | |
| { | |
| case ' ': | |
| case '\t': | |
| break; | |
| case '\n': | |
| case '\r': | |
| { | |
| if(peekAhead() == '\n') | |
| ++m_pos; | |
| m_columnOffset = m_pos; | |
| ++m_line; | |
| break; | |
| } | |
| default: | |
| return false; | |
| } | |
| ++m_pos; | |
| } | |
| return true; | |
| } | |
| Tokenizer::TokenType XQueryTokenizer::consumeWhitespace() | |
| { | |
| while(m_pos < m_length) | |
| { | |
| switch(peekCurrent()) | |
| { | |
| case ' ': | |
| case '\t': | |
| break; | |
| case '\n': | |
| case '\r': | |
| { | |
| /* We want to count \r\n as a single line break. */ | |
| if(peekAhead() == '\n') | |
| ++m_pos; | |
| m_columnOffset = m_pos; | |
| ++m_line; | |
| break; | |
| } | |
| case '(': | |
| { | |
| if(peekAhead() == ':') | |
| { | |
| m_pos += 2; /* Consume "(:" */ | |
| const TokenType comment = consumeComment(); | |
| if(comment == SUCCESS) | |
| continue; | |
| else | |
| return comment; | |
| } | |
| } | |
| default: | |
| return SUCCESS; | |
| } | |
| ++m_pos; | |
| } | |
| return END_OF_FILE; | |
| } | |
| char XQueryTokenizer::peekAhead(const int length) const | |
| { | |
| if(m_pos + length < m_length) | |
| return m_data.at(m_pos + length).toAscii(); | |
| else | |
| return 0; | |
| } | |
| Tokenizer::Token XQueryTokenizer::error() | |
| { | |
| return Token(ERROR); | |
| } | |
| bool XQueryTokenizer::isDigit(const char ch) | |
| { | |
| return ch >= '0' && ch <= '9'; | |
| } | |
| /* Replace with function in QXmlUtils. Write test cases for this. */ | |
| bool XQueryTokenizer::isNCNameStart(const QChar ch) | |
| { | |
| if(ch == QLatin1Char('_')) | |
| return true; | |
| switch(ch.category()) | |
| { | |
| case QChar::Letter_Lowercase: | |
| case QChar::Letter_Uppercase: | |
| case QChar::Letter_Other: | |
| case QChar::Letter_Titlecase: | |
| case QChar::Number_Letter: | |
| return true; | |
| default: | |
| return false; | |
| } | |
| } | |
| bool XQueryTokenizer::isNCNameBody(const QChar ch) | |
| { | |
| switch(ch.unicode()) | |
| { | |
| case '.': | |
| case '_': | |
| case '-': | |
| return true; | |
| } | |
| switch(ch.category()) | |
| { | |
| case QChar::Letter_Lowercase: | |
| case QChar::Letter_Uppercase: | |
| case QChar::Letter_Other: | |
| case QChar::Letter_Titlecase: | |
| case QChar::Number_Letter: | |
| case QChar::Mark_SpacingCombining: | |
| case QChar::Mark_Enclosing: | |
| case QChar::Mark_NonSpacing: | |
| case QChar::Letter_Modifier: | |
| case QChar::Number_DecimalDigit: | |
| return true; | |
| default: | |
| return false; | |
| } | |
| } | |
| bool XQueryTokenizer::isPhraseKeyword(const TokenType code) | |
| { | |
| switch(code) | |
| { | |
| /* Fallthrough all these. */ | |
| case CASTABLE: | |
| case CAST: | |
| case COPY_NAMESPACES: | |
| case DECLARE: | |
| case EMPTY: | |
| case MODULE: | |
| case IMPORT: | |
| case INSTANCE: | |
| case ORDER: | |
| case ORDERING: | |
| case XQUERY: | |
| case STABLE: | |
| case TREAT: | |
| return true; | |
| default: | |
| return false; | |
| } | |
| } | |
| bool XQueryTokenizer::isOperatorKeyword(const TokenType code) | |
| { | |
| switch(code) | |
| { | |
| /* Fallthrough all these. */ | |
| case AS: | |
| case ASCENDING: | |
| case AT: | |
| case CASE: | |
| case CAST: | |
| case CASTABLE: | |
| case EQ: | |
| case EXTERNAL: | |
| case GE: | |
| case G_EQ: | |
| case G_GT: | |
| case G_LT: | |
| case G_NE: | |
| case GT: | |
| case IN: | |
| case INHERIT: | |
| case INSTANCE: | |
| case IS: | |
| case ITEM: | |
| case LE: | |
| case LT: | |
| case NE: | |
| case NO_INHERIT: | |
| case NO_PRESERVE: | |
| case OF: | |
| case PRESERVE: | |
| case RETURN: | |
| case STABLE: | |
| case TO: | |
| case TREAT: | |
| return true; | |
| default: | |
| return false; | |
| }; | |
| } | |
| bool XQueryTokenizer::isTypeToken(const TokenType t) | |
| { | |
| switch(t) | |
| { | |
| /* Fallthrough all these. */ | |
| case ATTRIBUTE: | |
| case COMMENT: | |
| case DOCUMENT: | |
| case DOCUMENT_NODE: | |
| case ELEMENT: | |
| case ITEM: | |
| case NODE: | |
| case PROCESSING_INSTRUCTION: | |
| case SCHEMA_ATTRIBUTE: | |
| case SCHEMA_ELEMENT: | |
| case TEXT: | |
| return true; | |
| default: | |
| return false; | |
| } | |
| } | |
| Tokenizer::Token XQueryTokenizer::tokenizeNCNameOrQName() | |
| { | |
| const int start = m_pos; | |
| const Token t1 = tokenizeNCName(); | |
| if(t1.hasError()) | |
| return t1; | |
| if(peekCurrent() != ':' || peekAhead() == '=') | |
| return t1; | |
| ++m_pos; | |
| const Token t2 = tokenizeNCName(); | |
| if(t2.hasError()) | |
| return t2; | |
| else | |
| return Token(QNAME, m_data.mid(start, m_pos - start)); | |
| } | |
| Tokenizer::Token XQueryTokenizer::tokenizeNumberLiteral() | |
| { | |
| setState(Operator); | |
| const int startPos = m_pos; | |
| bool hasDot = false; | |
| bool isXPath20 = false; | |
| for(; m_pos < m_length; ++m_pos) | |
| { | |
| QChar ch(current()); | |
| char cell = ch.cell(); | |
| if(cell == 'e' || cell == 'E') | |
| { | |
| isXPath20 = true; | |
| ++m_pos; | |
| ch = current(); | |
| if(ch.row() != 0) | |
| break; | |
| cell = ch.cell(); | |
| if(cell == '+' || cell == '-') | |
| continue; | |
| } | |
| if(isNCNameStart(ch)) | |
| return error(); | |
| if(cell < '0' || cell > '9') | |
| { | |
| if(cell == '.' && !hasDot) | |
| hasDot = true; | |
| else | |
| break; | |
| } | |
| } | |
| return Token(isXPath20 ? XPATH2_NUMBER : NUMBER, m_data.mid(startPos, m_pos - startPos)); | |
| } | |
| QString XQueryTokenizer::tokenizeCharacterReference() | |
| { | |
| Q_ASSERT(peekCurrent() == '&'); | |
| const int theEnd = m_data.indexOf(QLatin1Char(';'), m_pos + 1); | |
| if(theEnd == -1) /* No ';' found, a syntax error. i18n. */ | |
| return QString(); | |
| QString content(m_data.mid(m_pos + 1, (theEnd - m_pos) - 1)); | |
| m_pos = theEnd; | |
| const QChar charRef(charForReference(content)); | |
| if(!charRef.isNull()) | |
| return charRef; | |
| else if(content.startsWith(QLatin1Char('#'))) | |
| { | |
| int base; | |
| /* It is only '#' or '#x'. */ | |
| if(content.length() < 2) | |
| return QString(); | |
| /* We got a hex number if it starts with 'x', otherwise it's a decimal. */ | |
| if(content.at(1) == QLatin1Char('x')) | |
| { | |
| base = 16; | |
| content = content.mid(2); /* Remove "#x". */ | |
| } | |
| else | |
| { | |
| base = 10; | |
| content = content.mid(1); /* Remove "#". */ | |
| } | |
| bool conversionOK = false; | |
| const int codepoint = content.toInt(&conversionOK, base); | |
| if(conversionOK) | |
| { | |
| const QChar ch(codepoint); | |
| if(ch.isNull()) | |
| { | |
| /* We likely have something which require surrogate pairs. */ | |
| QString result; | |
| result += QChar(QChar::highSurrogate(codepoint)); | |
| result += QChar(QChar::lowSurrogate(codepoint)); | |
| return result; | |
| } | |
| else | |
| return ch; | |
| } | |
| else | |
| return QString(); | |
| } | |
| else | |
| return QString(); | |
| } | |
| int XQueryTokenizer::scanUntil(const char *const content) | |
| { | |
| const int end = m_data.indexOf(QString::fromLatin1(content), m_pos); | |
| if(end == -1) | |
| return -1; | |
| else | |
| { | |
| const int len = end - m_pos; | |
| m_pos += len; | |
| return len; | |
| } | |
| } | |
| QChar XQueryTokenizer::charForReference(const QString &reference) | |
| { | |
| if(m_charRefs.isEmpty()) | |
| { | |
| /* Initialize. */ | |
| m_charRefs.reserve(5); | |
| m_charRefs.insert(QLatin1String("lt"), QLatin1Char('<')); | |
| m_charRefs.insert(QLatin1String("gt"), QLatin1Char('>')); | |
| m_charRefs.insert(QLatin1String("amp"), QLatin1Char('&')); | |
| m_charRefs.insert(QLatin1String("quot"), QLatin1Char('"')); | |
| m_charRefs.insert(QLatin1String("apos"), QLatin1Char('\'')); | |
| } | |
| return m_charRefs.value(reference); | |
| } | |
| Tokenizer::Token XQueryTokenizer::tokenizeStringLiteral() | |
| { | |
| const QChar delimiter(current()); | |
| /* We cannot unfortunately just scan and then do mid(), | |
| * since we can encounter character references. */ | |
| QString result; | |
| /* This is more likely than QString's default allocation. */ | |
| result.reserve(8); | |
| CharacterSkips skipEOLNormalization; | |
| /* Advance over the initial quote character. */ | |
| ++m_pos; | |
| for(; m_pos < m_length; ++m_pos) | |
| { | |
| const QChar c(current()); | |
| if(c == QLatin1Char('&')) | |
| { | |
| const QString charRef(tokenizeCharacterReference()); | |
| if(charRef.isNull()) | |
| return error(); | |
| else | |
| { | |
| skipEOLNormalization.insert(result.count()); | |
| result.append(charRef); | |
| } | |
| } | |
| else if(c == delimiter) | |
| { | |
| /* Maybe the escaping mechanism is used. For instance, "s""s" | |
| * has the value `s"s'. */ | |
| ++m_pos; | |
| if(current() == delimiter) /* Double quote. */ | |
| result += delimiter; | |
| else | |
| return Token(STRING_LITERAL, normalizeEOL(result, skipEOLNormalization)); | |
| } | |
| else | |
| result += c; | |
| } | |
| return error(); | |
| } | |
| Tokenizer::Token XQueryTokenizer::tokenizeNCName() | |
| { | |
| const int startPos = m_pos; | |
| if(m_pos < m_length && isNCNameStart(current())) | |
| { | |
| ++m_pos; | |
| for(; m_pos < m_length; ++m_pos) | |
| { | |
| if(!isNCNameBody(current())) | |
| break; | |
| } | |
| return Token(NCNAME, m_data.mid(startPos, m_pos - startPos)); | |
| } | |
| else | |
| return error(); | |
| } | |
| bool XQueryTokenizer::aheadEquals(const char *const chs, | |
| const int len, | |
| const int offset) const | |
| { | |
| Q_ASSERT(len > 0); | |
| Q_ASSERT(qstrlen(chs) == uint(len)); | |
| if(m_pos + len >= m_length) | |
| return false; | |
| for(int i = offset; i < (len + offset); ++i) | |
| { | |
| if(m_data.at(m_pos + i).toAscii() != chs[i - offset]) | |
| return false; | |
| } | |
| return true; | |
| } | |
| const TokenMap *XQueryTokenizer::lookupKeyword(const QString &keyword) | |
| { | |
| return TokenLookup::value(keyword.toAscii().constData(), keyword.length()); | |
| } | |
| XQueryTokenizer::State XQueryTokenizer::state() const | |
| { | |
| return m_state; | |
| } | |
| void XQueryTokenizer::setState(const State s) | |
| { | |
| m_state = s; | |
| } | |
| void XQueryTokenizer::pushState(const State s) | |
| { | |
| m_stateStack.push(s); | |
| } | |
| void XQueryTokenizer::pushState() | |
| { | |
| m_stateStack.push(m_state); | |
| } | |
| void XQueryTokenizer::popState() | |
| { | |
| /* QStack::pop() asserts if it's empty, so we need to check | |
| * it, since we might receive unbalanced curlies. */ | |
| if(!m_stateStack.isEmpty()) | |
| m_state = m_stateStack.pop(); | |
| } | |
| Tokenizer::Token XQueryTokenizer::nextToken() | |
| { | |
| switch(state()) | |
| { | |
| /* We want to skip or do special whitespace handling for these | |
| * states. So fallthrough all of the following. */ | |
| case AposAttributeContent: | |
| case Axis: | |
| case ElementContent: | |
| case EndTag: | |
| case Pragma: | |
| case PragmaContent: | |
| case ProcessingInstructionName: | |
| case QuotAttributeContent: | |
| case StartTag: | |
| case XMLComment: | |
| break; | |
| default: | |
| handleWhitespace(); | |
| } | |
| switch(state()) | |
| { | |
| case XMLSpaceDecl: | |
| /* Fallthrough. */ | |
| case NamespaceKeyword: | |
| { | |
| switch(peekCurrent()) | |
| { | |
| case ',': | |
| return tokenAndAdvance(COMMA); | |
| case '"': | |
| /* Fallthrough. */ | |
| case '\'': | |
| { | |
| setState(NamespaceDecl); | |
| return tokenizeStringLiteral(); | |
| } | |
| } | |
| const Token id(tokenizeNCName()); | |
| if(id.type != NCNAME) | |
| return id; | |
| const TokenMap *const keyword = lookupKeyword(id.value); | |
| if(keyword) | |
| { | |
| switch(keyword->token) | |
| { | |
| case INHERIT: | |
| /* Fallthrough. */ | |
| case NO_INHERIT: | |
| { | |
| setState(Default); | |
| break; | |
| } | |
| case NAMESPACE: | |
| { | |
| setState(NamespaceDecl); | |
| break; | |
| } | |
| case ORDERED: | |
| /* Fallthrough. */ | |
| case UNORDERED: | |
| /* Fallthrough. */ | |
| case STRIP: | |
| { | |
| setState(Default); | |
| break; | |
| } | |
| case PRESERVE: | |
| { | |
| if(state() != NamespaceKeyword) | |
| setState(Default); | |
| } | |
| default: | |
| break; | |
| } | |
| return Token(keyword->token); | |
| } | |
| else | |
| return id; | |
| Q_ASSERT(false); | |
| } | |
| case NamespaceDecl: | |
| { | |
| switch(peekCurrent()) | |
| { | |
| case '=': | |
| return tokenAndAdvance(G_EQ); | |
| case ';': | |
| return tokenAndChangeState(SEMI_COLON, Default); | |
| case '\'': | |
| /* Fallthrough. */ | |
| case '\"': | |
| return tokenizeStringLiteral(); | |
| } | |
| const Token nc(tokenizeNCName()); | |
| handleWhitespace(); | |
| const char pc = peekCurrent(); | |
| const TokenMap* const t = lookupKeyword(nc.value); | |
| if(pc == '\'' || (pc == '"' && t)) | |
| return tokenAndChangeState(t->token, Default, 0); | |
| else | |
| return nc; | |
| Q_ASSERT(false); | |
| } | |
| case Axis: | |
| { | |
| if(peekCurrent() == ':') | |
| { | |
| Q_ASSERT(peekAhead() == ':'); | |
| m_pos += 2; | |
| setState(AfterAxisSeparator); | |
| return Token(COLONCOLON); | |
| } | |
| /* Fallthrough. */ | |
| } | |
| case AfterAxisSeparator: | |
| /* Fallthrough. */ | |
| case Default: | |
| /* State Operator and state Default have a lot of tokens in common except | |
| * for minor differences. So we treat them the same way, and sprinkles logic | |
| * here and there to handle the small differences. */ | |
| /* Fallthrough. */ | |
| case Operator: | |
| { | |
| switch(peekCurrent()) | |
| { | |
| case '=': | |
| return tokenAndChangeState(G_EQ, Default); | |
| case '-': | |
| return tokenAndChangeState(MINUS, Default); | |
| case '+': | |
| return tokenAndChangeState(PLUS, Default); | |
| case '[': | |
| return tokenAndChangeState(LBRACKET, Default); | |
| case ']': | |
| return tokenAndChangeState(RBRACKET, Operator); | |
| case ',': | |
| return tokenAndChangeState(COMMA, Default); | |
| case ';': | |
| return tokenAndChangeState(SEMI_COLON, Default); | |
| case '$': | |
| return tokenAndChangeState(DOLLAR, VarName); | |
| case '|': | |
| return tokenAndChangeState(BAR, Default); | |
| case '?': | |
| return tokenAndChangeState(QUESTION, Operator); | |
| case ')': | |
| return tokenAndChangeState(RPAREN, Operator); | |
| case '@': | |
| return tokenAndChangeState(AT_SIGN, Default); | |
| /* Fallthrough all these. */ | |
| case '1': | |
| case '2': | |
| case '3': | |
| case '4': | |
| case '5': | |
| case '6': | |
| case '7': | |
| case '8': | |
| case '9': | |
| case '0': | |
| return tokenizeNumberLiteral(); | |
| case '.': | |
| { | |
| const char next = peekAhead(); | |
| if(next == '.') | |
| return tokenAndChangeState(DOTDOT, Operator, 2); | |
| /* .5 is allowed, as short form for 0.5: | |
| * <tt>[142] DecimalLiteral ::= ("." Digits) | (Digits "." [0-9]*)</tt> | |
| */ | |
| else if(isDigit(next)) | |
| return tokenizeNumberLiteral(); | |
| else | |
| return tokenAndChangeState(DOT, Operator); | |
| } | |
| case '\'': | |
| /* Fallthrough. */ | |
| case '"': | |
| { | |
| setState(Operator); | |
| return tokenizeStringLiteral(); | |
| } | |
| case '(': | |
| { | |
| if(peekAhead() == '#') | |
| return tokenAndChangeState(PRAGMA_START, Pragma, 2); | |
| else | |
| return tokenAndChangeState(LPAREN, Default); | |
| } | |
| case '*': | |
| { | |
| if(peekAhead() == ':') | |
| { | |
| m_pos += 2; /* Consume *:. */ | |
| const Token nc = tokenizeNCName(); | |
| if(nc.hasError()) | |
| return error(); | |
| else | |
| return tokenAndChangeState(ANY_PREFIX, nc.value, Operator); | |
| } | |
| else | |
| return tokenAndChangeState(STAR, state() == Default ? Operator : Default); | |
| } | |
| case ':': | |
| { | |
| switch(peekAhead()) | |
| { | |
| case '=': | |
| return tokenAndChangeState(ASSIGN, Default, 2); | |
| case ':': | |
| return tokenAndChangeState(COLONCOLON, Default, 2); | |
| default: | |
| return error(); | |
| } | |
| } | |
| case '!': | |
| { | |
| if(peekAhead() == '=') | |
| return tokenAndChangeState(G_NE, Default, 2); | |
| else | |
| return error(); | |
| } | |
| case '<': | |
| { | |
| switch(peekAhead()) | |
| { | |
| case '=': | |
| return tokenAndChangeState(G_LE, Default, 2); | |
| case '<': | |
| return tokenAndChangeState(PRECEDES, Default, 2); | |
| case '?': | |
| { | |
| pushState(Operator); | |
| return tokenAndChangeState(PI_START, ProcessingInstructionName, 2); | |
| } | |
| case '!': | |
| { | |
| if(aheadEquals("!--", 3)) | |
| { | |
| m_pos += 3; /* Consume "!--". */ | |
| pushState(Operator); | |
| return tokenAndChangeState(COMMENT_START, XMLComment); | |
| } | |
| /* Fallthrough. It's a syntax error, and this is a good way to report it. */ | |
| } | |
| default: | |
| { | |
| if((m_pos + 1) < m_length && isNCNameStart(m_data.at(m_pos + 1))) | |
| { | |
| /* We assume it's an element constructor. */ | |
| pushState(Operator); | |
| } | |
| return tokenAndChangeState(G_LT, state() == Operator ? Default : StartTag); | |
| } | |
| } | |
| } | |
| case '>': | |
| { | |
| switch(peekAhead()) | |
| { | |
| case '=': | |
| return tokenAndChangeState(G_GE, Default, 2); | |
| case '>': | |
| return tokenAndChangeState(FOLLOWS, Default, 2); | |
| default: | |
| return tokenAndChangeState(G_GT, Default); | |
| } | |
| } | |
| case '/': | |
| { | |
| if(peekAhead() == '/') | |
| return tokenAndChangeState(SLASHSLASH, Default, 2); | |
| else | |
| return tokenAndChangeState(SLASH, Default); | |
| } | |
| case '{': | |
| { | |
| pushState(Operator); | |
| return tokenAndChangeState(CURLY_LBRACE, Default); | |
| } | |
| case '}': | |
| { | |
| popState(); | |
| return tokenAndAdvance(CURLY_RBRACE); | |
| } | |
| } | |
| /* Ok. We're in state Default or Operator, and it wasn't a simple | |
| * character. */ | |
| const Token id(tokenizeNCName()); | |
| if(id.type != NCNAME) | |
| return id; | |
| const TokenMap *const keyword = lookupKeyword(id.value); | |
| if(state() == Operator) | |
| { | |
| if(keyword) | |
| { | |
| if(keyword->token == DEFAULT || keyword->token == ASCENDING || keyword->token == DESCENDING) | |
| setState(Operator); | |
| else if(keyword->token == RETURN) | |
| setState(Default); | |
| else if(isPhraseKeyword(keyword->token)) | |
| { | |
| const TokenType ws = consumeWhitespace(); | |
| if(ws == ERROR) | |
| return error(); | |
| const Token id2(tokenizeNCName()); | |
| const TokenMap *const keyword2 = lookupKeyword(id2.value); | |
| if(keyword2) | |
| { | |
| if(keyword->token == TREAT && keyword2->token == AS) | |
| setState(ItemType); | |
| else if (keyword->token == CAST || (keyword->token == CASTABLE && keyword2->token == AS) || keyword2->token == BY) | |
| setState(Default); | |
| m_tokenStack.push(Token(keyword2->token)); | |
| } | |
| else | |
| m_tokenStack.push(id2); | |
| return Token(keyword->token); | |
| } | |
| else | |
| { | |
| /* Such that we tokenize the second token in "empty greatest". */ | |
| if(keyword->token != EMPTY) | |
| setState(Default); | |
| } | |
| if(keyword->token == AS || keyword->token == CASE) | |
| setState(ItemType); | |
| return Token(keyword->token); | |
| } | |
| else | |
| return id; | |
| } | |
| Q_ASSERT(state() == Default || state() == Axis || state() == AfterAxisSeparator); | |
| /* | |
| * This is hard. Consider this: | |
| * | |
| * Valid: child ::nameTest | |
| * Valid: child:: nameTest | |
| * Syntax Error: child :localName | |
| * Syntax Error: child: localName | |
| * | |
| * Consider "child ::name". Right now, we're here: | |
| * ^ | |
| * We don't know whether "child" is a prefix and hence the whitespace is invalid, | |
| * or whether it's an axis and hence skippable. */ | |
| { | |
| const int wsLength = peekForColonColon(); | |
| /* We cannot call handleWhitespace() because it returns on | |
| * END_OF_FILE, and we have parsed up keyword, and we need to | |
| * deal with that. | |
| * | |
| * If we have a colon colon, which means the whitespace is | |
| * allowed, we skip it. */ | |
| if(wsLength != -1) | |
| m_pos += wsLength; | |
| } | |
| /* Handle name tests. */ | |
| if(peekCurrent() == ':') | |
| { | |
| switch(peekAhead()) | |
| { | |
| case '=': | |
| return id; | |
| case '*': | |
| { | |
| m_pos += 2; | |
| return tokenAndChangeState(ANY_LOCAL_NAME, id.value, Operator); | |
| } | |
| case ':': | |
| { | |
| /* We have an axis. */ | |
| setState(Axis); | |
| return keyword ? Token(keyword->token) : id; | |
| } | |
| default: | |
| { | |
| /* It's a QName. */ | |
| ++m_pos; /* Consume the colon. */ | |
| const Token id2(tokenizeNCName()); | |
| if(id2.type != NCNAME) | |
| { | |
| --m_pos; | |
| return id; | |
| } | |
| setState(Operator); | |
| const int qNameLen = id.value.length() + id2.value.length() + 1; | |
| return Token(QNAME, m_data.mid(m_pos - qNameLen, qNameLen)); | |
| } | |
| } | |
| } | |
| if(!keyword || isOperatorKeyword(keyword->token)) | |
| { | |
| setState(Operator); | |
| return id; | |
| } | |
| const TokenType ws = consumeWhitespace(); | |
| if(ws == ERROR) // TODO this should test for success. Write test. | |
| return Token(ERROR); | |
| if(atEnd()) | |
| { | |
| setState(Operator); | |
| return id; | |
| } | |
| /* Let the if-body apply for constructors, and node type tests. */ | |
| if(isTypeToken(keyword->token) || | |
| keyword->token == TYPESWITCH || | |
| keyword->token == ORDERED || | |
| keyword->token == UNORDERED || | |
| keyword->token == IF) | |
| { | |
| switch(peekCurrent()) | |
| { | |
| case '(': | |
| { | |
| // TODO See if we can remove DOCUMENT from isTypeToken. | |
| if(isTypeToken(keyword->token) && keyword->token != DOCUMENT) | |
| { | |
| m_tokenStack.push(Token(LPAREN)); | |
| ++m_pos; /* Consume '('. */ | |
| pushState(Operator); | |
| if(keyword->token == PROCESSING_INSTRUCTION) | |
| setState(KindTestForPI); | |
| else | |
| setState(KindTest); | |
| return Token(keyword->token); | |
| } | |
| else if(keyword->token == TYPESWITCH || keyword->token == IF) | |
| return Token(keyword->token); | |
| else /* It's a function call. */ | |
| return id; | |
| } | |
| case '{': | |
| { | |
| m_tokenStack.push(Token(CURLY_LBRACE)); | |
| ++m_pos; /* Consume '{'. */ | |
| pushState(Operator); | |
| /* Stay in state Default. */ | |
| return Token(keyword->token); | |
| } | |
| default: | |
| { | |
| /* We have read in a token which is for instance | |
| * "return", and now it can be an element | |
| * test("element") a node kind test("element()"), or a | |
| * computed element constructor("element name {..."). | |
| * We need to do a two-token lookahead here, because | |
| * "element return" can be an element test followed by | |
| * the return keyword, but it can also be an element | |
| * constructor("element return {"). */ | |
| if(isNCNameStart(current())) | |
| { | |
| const int currentPos = m_pos; | |
| const Token token2 = tokenizeNCNameOrQName(); | |
| if(token2.hasError()) | |
| return token2; | |
| handleWhitespace(); | |
| if(peekCurrent() == '{') | |
| { | |
| /* An element constructor. */ | |
| m_tokenStack.push(token2); | |
| return Token(keyword->token); | |
| } | |
| /* We jump back in the stream, we need to tokenize token2 according | |
| * to the state. */ | |
| m_pos = currentPos; | |
| setState(Operator); | |
| return Token(NCNAME, QLatin1String(keyword->name)); | |
| } | |
| } | |
| } | |
| } | |
| if(peekCurrent() == '$') | |
| { | |
| setState(VarName); | |
| return Token(keyword->token); | |
| } | |
| /* It's not a node type, it's not the typeswitch expression, but it is a function callsite. */ | |
| if(peekCurrent() == '(') | |
| return id; | |
| else if(peekCurrent() == '{' && keyword->token == VALIDATE) | |
| return Token(keyword->token); | |
| if(!isNCNameStart(current())) | |
| { | |
| setState(Operator); | |
| return id; | |
| } | |
| const Token id2(tokenizeNCName()); | |
| const TokenMap *const keyword2 = lookupKeyword(id2.value); | |
| if(!keyword2) | |
| { | |
| /* It's a syntax error. All cases of two subsequent ncnames are keywords(e.g, declarations). */ | |
| setState(Operator); | |
| return id; | |
| } | |
| switch(keyword->token) | |
| { | |
| case DECLARE: | |
| { | |
| switch(keyword2->token) | |
| { | |
| case VARIABLE: | |
| /* Fallthrough. */ | |
| case FUNCTION: | |
| { | |
| m_tokenStack.push(Token(keyword2->token)); | |
| setState(Default); | |
| return Token(keyword->token); | |
| } | |
| case OPTION: | |
| { | |
| m_tokenStack.push(Token(keyword2->token)); | |
| setState(Default); | |
| return Token(keyword->token); | |
| } | |
| case COPY_NAMESPACES: | |
| /* Fallthrough. */ | |
| case ORDERING: | |
| { | |
| m_tokenStack.push(Token(keyword2->token)); | |
| setState(NamespaceKeyword); | |
| return Token(keyword->token); | |
| } | |
| case CONSTRUCTION: | |
| { | |
| // TODO identical to CONSTRUCTION? | |
| m_tokenStack.push(Token(keyword2->token)); | |
| setState(Operator); | |
| return Token(keyword->token); | |
| } | |
| case NAMESPACE: | |
| /* Fallthrough. */ | |
| case BASEURI: | |
| { | |
| m_tokenStack.push(Token(keyword2->token)); | |
| setState(NamespaceDecl); | |
| return Token(keyword->token); | |
| } | |
| case BOUNDARY_SPACE: | |
| { | |
| m_tokenStack.push(Token(keyword2->token)); | |
| setState(XMLSpaceDecl); | |
| return Token(keyword->token); | |
| } | |
| case DEFAULT: | |
| { | |
| m_tokenStack.push(Token(keyword2->token)); | |
| const TokenType ws2 = consumeWhitespace(); | |
| if(ws2 != SUCCESS) | |
| { | |
| m_tokenStack.prepend(Token(ws2)); | |
| return Token(keyword->token); | |
| } | |
| const Token id3(tokenizeNCName()); | |
| if(id3.type != NCNAME) | |
| { | |
| m_tokenStack.prepend(id3); | |
| return Token(keyword->token); | |
| } | |
| const TokenMap *const keyword3 = lookupKeyword(id3.value); | |
| if(!keyword3) | |
| { | |
| m_tokenStack.prepend(id3); | |
| return Token(keyword->token); | |
| } | |
| else | |
| { | |
| m_tokenStack.prepend(Token(keyword3->token)); | |
| if(keyword3->token == ORDER) | |
| setState(Operator); | |
| else | |
| setState(NamespaceDecl); | |
| } | |
| return Token(keyword->token); | |
| } | |
| default: | |
| { | |
| m_tokenStack.push(Token(keyword2->token)); | |
| setState(Default); | |
| return id; | |
| } | |
| } | |
| } | |
| case XQUERY: | |
| { | |
| m_tokenStack.push(Token(keyword2->token)); | |
| if(keyword2->token == VERSION) | |
| { | |
| setState(NamespaceDecl); | |
| return Token(keyword->token); | |
| } | |
| else | |
| { | |
| setState(Operator); | |
| return id; | |
| } | |
| } | |
| case IMPORT: | |
| { | |
| m_tokenStack.push(Token(keyword2->token)); | |
| switch(keyword2->token) | |
| { | |
| case SCHEMA: | |
| /* Fallthrough. */ | |
| case MODULE: | |
| { | |
| setState(NamespaceKeyword); | |
| return Token(keyword->token); | |
| } | |
| default: | |
| { | |
| setState(Operator); | |
| return id; | |
| } | |
| } | |
| } | |
| case VALIDATE: | |
| { | |
| m_tokenStack.push(Token(keyword2->token)); | |
| switch(keyword2->token) | |
| { | |
| case LAX: | |
| case STRICT: | |
| { | |
| pushState(Operator); | |
| return Token(keyword->token); | |
| } | |
| default: | |
| { | |
| setState(Operator); | |
| return id; | |
| } | |
| } | |
| } | |
| default: | |
| { | |
| m_tokenStack.push(Token(keyword2->token)); | |
| setState(Operator); | |
| return id; | |
| } | |
| } | |
| Q_ASSERT(false); | |
| } | |
| case VarName: | |
| { | |
| if(peekCurrent() == '$') | |
| return tokenAndAdvance(DOLLAR); | |
| setState(Operator); | |
| return tokenizeNCNameOrQName(); | |
| Q_ASSERT(false); | |
| } | |
| case ItemType: | |
| { | |
| switch(peekCurrent()) | |
| { | |
| case '(': | |
| return tokenAndChangeState(LPAREN, KindTest); | |
| case '$': | |
| return tokenAndChangeState(DOLLAR, VarName); | |
| } | |
| const Token name(tokenizeNCNameOrQName()); | |
| if(name.hasError()) | |
| return error(); | |
| else if(name.type == QNAME) | |
| { | |
| setState(OccurrenceIndicator); | |
| return name; | |
| } | |
| else | |
| { | |
| const TokenMap *const keyword = lookupKeyword(name.value); | |
| if(keyword) | |
| { | |
| pushState(OccurrenceIndicator); | |
| return Token(keyword->token); | |
| } | |
| else | |
| { | |
| setState(Default); | |
| return name; | |
| } | |
| } | |
| Q_ASSERT(false); | |
| } | |
| case KindTest: | |
| { | |
| switch(peekCurrent()) | |
| { | |
| case ')': | |
| { | |
| popState(); | |
| return tokenAndAdvance(RPAREN); | |
| } | |
| case '(': | |
| return tokenAndAdvance(LPAREN); | |
| case ',': | |
| return tokenAndAdvance(COMMA); | |
| case '*': | |
| return tokenAndAdvance(STAR); | |
| case '?': | |
| return tokenAndAdvance(QUESTION); | |
| case '\'': | |
| /* Fallthrough. */ | |
| case '"': | |
| return tokenizeStringLiteral(); | |
| } | |
| const Token nc(tokenizeNCNameOrQName()); | |
| if(nc.hasError()) | |
| return nc; | |
| const TokenType ws = consumeWhitespace(); | |
| if(ws == ERROR) | |
| return error(); | |
| if(peekCurrent() == '(') | |
| { | |
| const TokenMap *const keyword = lookupKeyword(nc.value); | |
| if(keyword) | |
| { | |
| pushState(KindTest); | |
| return Token(keyword->token); | |
| } | |
| else | |
| return nc; | |
| } | |
| else | |
| return nc; | |
| Q_ASSERT(false); | |
| } | |
| case KindTestForPI: | |
| { | |
| switch(peekCurrent()) | |
| { | |
| case ')': | |
| { | |
| popState(); | |
| return tokenAndAdvance(RPAREN); | |
| } | |
| case '\'': | |
| /* Fallthrough. */ | |
| case '"': | |
| return tokenizeStringLiteral(); | |
| default: | |
| return tokenizeNCName(); | |
| } | |
| Q_ASSERT(false); | |
| } | |
| case OccurrenceIndicator: | |
| { | |
| switch(peekCurrent()) | |
| { | |
| case '?': | |
| return tokenAndChangeState(QUESTION, Operator); | |
| case '*': | |
| return tokenAndChangeState(STAR, Operator); | |
| case '+': | |
| return tokenAndChangeState(PLUS, Operator); | |
| default: | |
| { | |
| setState(Operator); | |
| return nextToken(); | |
| } | |
| } | |
| Q_ASSERT(false); | |
| } | |
| case XQueryVersion: | |
| { | |
| switch(peekCurrent()) | |
| { | |
| case '\'': | |
| /* Fallthrough. */ | |
| case '"': | |
| return tokenizeStringLiteral(); | |
| case ';': | |
| return tokenAndChangeState(SEMI_COLON, Default); | |
| } | |
| const Token id(tokenizeNCName()); | |
| if(id.type != NCNAME) | |
| return id; | |
| const TokenMap *const keyword = lookupKeyword(id.value); | |
| if(keyword) | |
| return tokenAndChangeState(keyword->token, Default); | |
| else | |
| return id; | |
| Q_ASSERT(false); | |
| } | |
| case StartTag: | |
| { | |
| if(peekAhead(-1) == '<') | |
| { | |
| if(current().isSpace()) | |
| return Token(ERROR); | |
| } | |
| else | |
| { | |
| if(consumeRawWhitespace()) | |
| return Token(END_OF_FILE); | |
| } | |
| switch(peekCurrent()) | |
| { | |
| case '/': | |
| { | |
| if(peekAhead() == '>') | |
| { | |
| m_pos += 2; | |
| if(m_scanOnly) | |
| return Token(POSITION_SET); | |
| else | |
| { | |
| popState(); | |
| return Token(QUICK_TAG_END); | |
| } | |
| } | |
| else | |
| return error(); | |
| } | |
| case '>': | |
| { | |
| if(m_scanOnly) | |
| return tokenAndChangeState(POSITION_SET, StartTag); | |
| else | |
| return tokenAndChangeState(G_GT, ElementContent); | |
| } | |
| case '=': | |
| return tokenAndAdvance(G_EQ); | |
| case '\'': | |
| return tokenAndChangeState(APOS, AposAttributeContent); | |
| case '"': | |
| return tokenAndChangeState(QUOTE, QuotAttributeContent); | |
| default: | |
| return tokenizeNCNameOrQName(); | |
| } | |
| Q_ASSERT(false); | |
| } | |
| case AposAttributeContent: | |
| /* Fallthrough. */ | |
| case QuotAttributeContent: | |
| { | |
| const QChar sep(state() == AposAttributeContent ? QLatin1Char('\'') : QLatin1Char('"')); | |
| QString result; | |
| result.reserve(20); | |
| if(m_scanOnly) | |
| { | |
| int stack = 0; | |
| return attributeAsRaw(sep, stack, m_pos, true, result); | |
| } | |
| Q_ASSERT(!m_scanOnly); | |
| while(true) | |
| { | |
| if(atEnd()) | |
| { | |
| /* In the case that the XSL-T tokenizer invokes us with | |
| * default state QuotAttributeContent, we need to be able | |
| * to return a single string, in case that is all we have | |
| * accumulated. */ | |
| if(result.isEmpty()) | |
| return Token(END_OF_FILE); | |
| else | |
| return Token(STRING_LITERAL, result); | |
| } | |
| const QChar curr(current()); | |
| if(curr == sep) | |
| { | |
| if(m_pos + 1 == m_length) | |
| return Token(END_OF_FILE); | |
| if(m_data.at(m_pos + 1) == sep) | |
| { | |
| /* The quoting mechanism was used. */ | |
| m_pos += 2; | |
| result.append(sep); | |
| continue; | |
| } | |
| const QChar next(m_data.at(m_pos + 1)); | |
| if(!next.isSpace() && next != QLatin1Char('/') && next != QLatin1Char('>')) | |
| return Token(ERROR); // i18n Space must separate attributes | |
| else if(result.isEmpty()) | |
| { | |
| return tokenAndChangeState(state() == AposAttributeContent ? APOS : QUOTE, | |
| StartTag, 1); | |
| } | |
| else | |
| { | |
| /* Don't consume the sep, but leave it so we next time return a token for it. */ | |
| return Token(STRING_LITERAL, result); | |
| } | |
| ++m_pos; | |
| continue; | |
| } | |
| else if(curr == QLatin1Char('{')) | |
| { | |
| if(m_pos + 1 == m_length) | |
| return Token(END_OF_FILE); | |
| else if(peekAhead() == '{') | |
| { | |
| ++m_pos; | |
| result.append(QLatin1Char('{')); | |
| } | |
| else | |
| { | |
| if(result.isEmpty()) | |
| { | |
| /* The Attribute Value Template appeared directly in the attribute. */ | |
| pushState(); | |
| return tokenAndChangeState(CURLY_LBRACE, Default); | |
| } | |
| else | |
| { | |
| /* We don't advance, keep '{' as next token. */ | |
| return Token(STRING_LITERAL, result); | |
| } | |
| } | |
| } | |
| else if(curr == QLatin1Char('}')) | |
| { | |
| if(m_pos + 1 == m_length) | |
| return Token(END_OF_FILE); | |
| else if(peekAhead() == '}') | |
| { | |
| ++m_pos; | |
| result.append(QLatin1Char('}')); | |
| } | |
| else | |
| return Token(ERROR); | |
| } | |
| else if(curr == QLatin1Char('&')) | |
| { | |
| const QString ret(tokenizeCharacterReference()); | |
| if(ret.isNull()) | |
| return Token(ERROR); | |
| else | |
| result.append(ret); | |
| } | |
| else if(curr == QLatin1Char('<')) | |
| return Token(STRING_LITERAL, result); | |
| else | |
| { | |
| /* See Extensible Markup Language (XML) 1.0 (Fourth Edition), | |
| * 3.3.3 Attribute-Value Normalization. | |
| * | |
| * However, it is complicated a bit by that AVN is defined on top of | |
| * EOL normalization and we do those two in one go here. */ | |
| switch(curr.unicode()) | |
| { | |
| case 0xD: | |
| { | |
| if(peekAhead() == '\n') | |
| { | |
| result.append(QLatin1Char(' ')); | |
| ++m_pos; | |
| break; | |
| } | |
| } | |
| case 0xA: | |
| /* Fallthrough. */ | |
| case 0x9: | |
| { | |
| result.append(QLatin1Char(' ')); | |
| break; | |
| } | |
| default: | |
| result.append(curr); | |
| } | |
| } | |
| ++m_pos; | |
| } | |
| Q_ASSERT(false); | |
| } | |
| case ElementContent: | |
| { | |
| QString result; | |
| result.reserve(20); | |
| /* Whether the text node, result, may be whitespace only. Character references | |
| * and CDATA sections disables that. */ | |
| bool mayBeWS = true; | |
| CharacterSkips skipEOLNormalization; | |
| while(true) | |
| { | |
| if(atEnd()) | |
| return Token(END_OF_FILE); | |
| switch(peekCurrent()) | |
| { | |
| case '<': | |
| { | |
| if(!result.isEmpty() && peekAhead(2) != '[') | |
| { | |
| /* We encountered the end, and it was not a CDATA section. */ | |
| /* We don't advance. Next time we'll handle the <... stuff. */ | |
| return Token(mayBeWS ? STRING_LITERAL : NON_BOUNDARY_WS, normalizeEOL(result, skipEOLNormalization)); | |
| } | |
| ++m_pos; | |
| if(atEnd()) | |
| return Token(END_OF_FILE); | |
| const QChar ahead(current()); | |
| if(ahead.isSpace()) | |
| return error(); | |
| else if(ahead == QLatin1Char('/')) | |
| { | |
| if(m_pos + 1 == m_length) | |
| return Token(END_OF_FILE); | |
| else if(m_data.at(m_pos + 1).isSpace()) | |
| return error(); | |
| else | |
| return tokenAndChangeState(BEGIN_END_TAG, EndTag); | |
| } | |
| else if(isNCNameStart(ahead)) | |
| { | |
| pushState(); | |
| return tokenAndChangeState(G_LT, StartTag, 0); | |
| } | |
| else if(aheadEquals("!--", 3, 0)) | |
| { | |
| pushState(); | |
| m_pos += 3; | |
| return tokenAndChangeState(COMMENT_START, XMLComment, 0); | |
| } | |
| else if(aheadEquals("![CDATA[", 8, 0)) | |
| { | |
| mayBeWS = false; | |
| m_pos += 8; | |
| const int start = m_pos; | |
| const int len = scanUntil("]]>"); | |
| if(len == -1) | |
| return Token(END_OF_FILE); | |
| m_pos += 2; /* Consume "]]>". Note that m_pos is on '!'. */ | |
| result.append(m_data.mid(start, len)); | |
| break; | |
| } | |
| else if(ahead == QLatin1Char('?')) | |
| { | |
| pushState(); | |
| return tokenAndChangeState(PI_START, ProcessingInstructionName); | |
| } | |
| else | |
| return Token(G_LT); | |
| } | |
| case '&': | |
| { | |
| const QString ret(tokenizeCharacterReference()); | |
| if(ret.isNull()) | |
| return Token(ERROR); | |
| else | |
| { | |
| skipEOLNormalization.insert(result.count()); | |
| result.append(ret); | |
| mayBeWS = false; | |
| break; | |
| } | |
| } | |
| case '{': | |
| { | |
| // TODO remove this check, also below. | |
| if(m_pos + 1 == m_length) | |
| return Token(END_OF_FILE); | |
| else if(peekAhead() == '{') | |
| { | |
| ++m_pos; | |
| result.append(QLatin1Char('{')); | |
| } | |
| else | |
| { | |
| if(result.isEmpty()) | |
| { | |
| pushState(); | |
| return tokenAndChangeState(CURLY_LBRACE, Default); | |
| } | |
| else | |
| { | |
| /* We don't advance here. */ | |
| return Token(mayBeWS ? STRING_LITERAL : NON_BOUNDARY_WS, normalizeEOL(result, skipEOLNormalization)); | |
| } | |
| } | |
| break; | |
| } | |
| case '}': | |
| { | |
| if(m_pos + 1 == m_length) | |
| return Token(END_OF_FILE); | |
| else if(peekAhead() == '}') | |
| { | |
| ++m_pos; | |
| result.append(QLatin1Char('}')); | |
| } | |
| else | |
| { | |
| /* This is a parse error, and the grammar won't be able | |
| * to reduce this CURLY_RBRACE. */ | |
| return tokenAndChangeState(CURLY_RBRACE, Default); | |
| } | |
| break; | |
| } | |
| case '\n': | |
| { | |
| /* We want to translate \r\n into \n. */ | |
| if(peekAhead(-1) == '\r') | |
| break; | |
| /* else, fallthrough. */ | |
| } | |
| case '\r': | |
| { | |
| result.append(QLatin1Char('\n')); | |
| break; | |
| } | |
| default: | |
| { | |
| result.append(current()); | |
| break; | |
| } | |
| } | |
| ++m_pos; | |
| } | |
| Q_ASSERT(false); | |
| } | |
| case ProcessingInstructionName: | |
| { | |
| const int start = m_pos; | |
| while(true) | |
| { | |
| ++m_pos; | |
| if(m_pos >= m_length) | |
| return Token(END_OF_FILE); | |
| const QChar next(current()); | |
| if(next.isSpace() || next == QLatin1Char('?')) | |
| { | |
| return tokenAndChangeState(PI_TARGET, m_data.mid(start, m_pos - start), | |
| ProcessingInstructionContent); | |
| } | |
| } | |
| Q_ASSERT(false); | |
| } | |
| case ProcessingInstructionContent: | |
| { | |
| /* Consume whitespace between the name and the content. */ | |
| if(consumeRawWhitespace()) | |
| return Token(END_OF_FILE); | |
| const int start = m_pos; | |
| const int len = scanUntil("?>"); | |
| if(len == -1) | |
| return Token(END_OF_FILE); | |
| else | |
| { | |
| m_pos += 2; /* Consume "?>" */ | |
| popState(); | |
| return Token(PI_CONTENT, normalizeEOL(m_data.mid(start, len), CharacterSkips())); | |
| } | |
| Q_ASSERT(false); | |
| } | |
| case EndTag: | |
| { | |
| if(consumeRawWhitespace()) | |
| return END_OF_FILE; | |
| if(peekCurrent() == '>') | |
| { | |
| popState(); | |
| return tokenAndAdvance(G_GT); | |
| } | |
| else | |
| return tokenizeNCNameOrQName(); | |
| Q_ASSERT(false); | |
| } | |
| case XMLComment: | |
| { | |
| const int start = m_pos; | |
| const int len = scanUntil("--"); | |
| if(len == -1) | |
| return END_OF_FILE; | |
| else | |
| { | |
| m_pos += 2; /* Consume "--". */ | |
| popState(); | |
| if(peekCurrent() == '>') | |
| { | |
| ++m_pos; | |
| return Token(COMMENT_CONTENT, normalizeEOL(m_data.mid(start, len), CharacterSkips())); | |
| } | |
| else | |
| return error(); | |
| } | |
| Q_ASSERT(false); | |
| } | |
| case Pragma: | |
| { | |
| /* Consume whitespace. */ | |
| if(consumeRawWhitespace()) | |
| return Token(END_OF_FILE); | |
| setState(PragmaContent); | |
| return tokenizeNCNameOrQName(); | |
| } | |
| case PragmaContent: | |
| { | |
| QString result; | |
| result.reserve(20); | |
| const bool hasWS = m_pos < m_length && current().isSpace(); | |
| /* Consume all whitespace up to the pragma content(if any). */ | |
| if(consumeRawWhitespace()) | |
| return Token(END_OF_FILE); | |
| if(peekCurrent() == '#' && peekAhead() == ')') | |
| { | |
| /* We reached the end, and there's no pragma content. */ | |
| return tokenAndChangeState(PRAGMA_END, Default, 2); | |
| } | |
| else if(!hasWS) | |
| { | |
| /* A separating space is required if there's pragma content. */ | |
| return error(); /* i18n */ | |
| } | |
| const int start = m_pos; | |
| const int len = scanUntil("#)"); | |
| if(len == -1) | |
| return Token(END_OF_FILE); | |
| return Token(STRING_LITERAL, m_data.mid(start, len)); | |
| Q_ASSERT(false); | |
| } | |
| } | |
| Q_ASSERT(false); | |
| return error(); | |
| } | |
| Tokenizer::Token XQueryTokenizer::attributeAsRaw(const QChar sep, | |
| int &sepStack, | |
| const int startPos, | |
| const bool aInLiteral, | |
| QString &result) | |
| { | |
| bool inLiteral = aInLiteral; | |
| const char otherSep = (sep == QLatin1Char('"') ? '\'' : '"'); | |
| while(true) | |
| { | |
| if(atEnd()) | |
| return END_OF_FILE; | |
| if(peekCurrent() == sep.unicode()) | |
| { | |
| if(inLiteral) | |
| inLiteral = false; | |
| else | |
| inLiteral = true; | |
| if(peekAhead() == sep.unicode()) | |
| { | |
| /* The quoting mechanism was used. */ | |
| result.append(current()); | |
| m_pos += 2; | |
| continue; | |
| } | |
| else | |
| { | |
| /* Don't consume the separator, such that we | |
| * return a token for it next time. */ | |
| if(m_pos == startPos) | |
| { | |
| ++m_pos; | |
| setState(StartTag); | |
| return Token(sep == QLatin1Char('"') ? QUOTE : APOS); | |
| } | |
| if(sepStack == 0) | |
| { | |
| return Token(STRING_LITERAL, result); | |
| } | |
| else | |
| { | |
| result.append(current()); | |
| ++m_pos; | |
| continue; | |
| } | |
| } | |
| } | |
| else if(peekCurrent() == '&') | |
| { | |
| const QString ret(tokenizeCharacterReference()); | |
| if(ret.isNull()) | |
| return Token(ERROR); | |
| else | |
| { | |
| result.append(ret); | |
| ++m_pos; | |
| continue; | |
| } | |
| } | |
| else if(peekCurrent() == otherSep) | |
| { | |
| result.append(current()); | |
| ++m_pos; | |
| if(peekCurrent() == otherSep) | |
| ++m_pos; | |
| if(inLiteral) | |
| inLiteral = false; | |
| else | |
| inLiteral = true; | |
| continue; | |
| } | |
| else if(peekCurrent() == '{') | |
| { | |
| result.append(current()); | |
| if(peekAhead() == '{') | |
| { | |
| m_pos += 2; | |
| continue; | |
| } | |
| else | |
| { | |
| ++m_pos; | |
| ++sepStack; | |
| const Token t(attributeAsRaw(sep, sepStack, startPos, false, result)); | |
| if(t.type != SUCCESS) | |
| return t; | |
| } | |
| } | |
| else if(peekCurrent() == '}') | |
| { | |
| if(inLiteral && peekAhead() == '}') | |
| { | |
| result.append(current()); | |
| m_pos += 2; | |
| continue; | |
| } | |
| else | |
| { | |
| ++m_pos; | |
| --sepStack; | |
| return Token(SUCCESS); /* The return value is arbitrary. */ | |
| } | |
| } | |
| else | |
| { | |
| result.append(current()); | |
| ++m_pos; | |
| } | |
| } | |
| } | |
| Tokenizer::Token XQueryTokenizer::nextToken(YYLTYPE *const sourceLocator) | |
| { | |
| sourceLocator->first_line = m_line; | |
| sourceLocator->first_column = m_pos - m_columnOffset + 1; /* Plus 1, since m_pos is 0-based. */ | |
| if(m_tokenStack.isEmpty()) | |
| return nextToken(); | |
| else | |
| { | |
| const Token retval(m_tokenStack.pop()); | |
| switch(retval.type) | |
| { | |
| case MODULE: | |
| /* Fallthrough.*/ | |
| case SCHEMA: | |
| /* Fallthrough.*/ | |
| case COPY_NAMESPACES: | |
| { | |
| setState(NamespaceKeyword); | |
| break; | |
| } | |
| case VERSION: | |
| { | |
| setState(XQueryVersion); | |
| break; | |
| } | |
| case AS: | |
| /* Fallthrough. */ | |
| case OF: | |
| { | |
| setState(ItemType); | |
| break; | |
| } | |
| default: | |
| { | |
| if(isOperatorKeyword(retval.type)) | |
| setState(Default); | |
| break; | |
| } | |
| }; | |
| return retval; | |
| } | |
| } | |
| int XQueryTokenizer::commenceScanOnly() | |
| { | |
| m_scanOnly = true; | |
| return m_pos; | |
| } | |
| void XQueryTokenizer::resumeTokenizationFrom(const int pos) | |
| { | |
| m_scanOnly = false; | |
| m_pos = pos; | |
| } | |
| void XQueryTokenizer::setParserContext(const ParserContext::Ptr &) | |
| { | |
| } | |
| #undef handleWhitespace | |
| } // namespace QPatternist | |
| QT_END_NAMESPACE |