Pull the bulk of Lexer::MeasureTokenLength() out into a new function,
Lexer::getRawToken().
No functionality change.
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@171771 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/include/clang/Lex/Lexer.h b/include/clang/Lex/Lexer.h
index b57be3a..d36189f 100644
--- a/include/clang/Lex/Lexer.h
+++ b/include/clang/Lex/Lexer.h
@@ -273,6 +273,12 @@
const SourceManager &SM,
const LangOptions &LangOpts);
+ /// \brief Relex the token at the specified location.
+ /// \returns true if there was a failure, false on success.
+ static bool getRawToken(SourceLocation Loc, Token &Result,
+ const SourceManager &SM,
+ const LangOptions &LangOpts);
+
/// \brief Given a location any where in a source buffer, find the location
/// that corresponds to the beginning of the token in which the original
/// source location lands.
diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h
index fb915bb..9d7db5d 100644
--- a/include/clang/Lex/Preprocessor.h
+++ b/include/clang/Lex/Preprocessor.h
@@ -958,6 +958,12 @@
SmallVectorImpl<char> &Buffer,
bool *Invalid = 0) const;
+ /// \brief Relex the token at the specified location.
+ /// \returns true if there was a failure, false on success.
+ bool getRawToken(SourceLocation Loc, Token &Result) {
+ return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts);
+ }
+
/// getSpellingOfSingleCharacterNumericConstant - Tok is a numeric constant
/// with length 1, return the character.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok,
diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index 1bcff98..15b1061 100644
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -414,6 +414,17 @@
unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
const SourceManager &SM,
const LangOptions &LangOpts) {
+ Token TheTok;
+ if (getRawToken(Loc, TheTok, SM, LangOpts))
+ return 0;
+ return TheTok.getLength();
+}
+
+/// \brief Relex the token at the specified location.
+/// \returns true if there was a failure, false on success.
+bool Lexer::getRawToken(SourceLocation Loc, Token &Result,
+ const SourceManager &SM,
+ const LangOptions &LangOpts) {
// TODO: this could be special cased for common tokens like identifiers, ')',
// etc to make this faster, if it mattered. Just look at StrData[0] to handle
// all obviously single-char tokens. This could use
@@ -427,20 +438,19 @@
bool Invalid = false;
StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
if (Invalid)
- return 0;
+ return true;
const char *StrData = Buffer.data()+LocInfo.second;
if (isWhitespace(StrData[0]))
- return 0;
+ return true;
// Create a lexer starting at the beginning of this token.
Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts,
Buffer.begin(), StrData, Buffer.end());
TheLexer.SetCommentRetentionState(true);
- Token TheTok;
- TheLexer.LexFromRawLexer(TheTok);
- return TheTok.getLength();
+ TheLexer.LexFromRawLexer(Result);
+ return false;
}
static SourceLocation getBeginningOfFileToken(SourceLocation Loc,
