Introduced preprocessing token types. This fixes a bug where invalid tokens inside excluded conditional block may report diagnostics. Now we let the invalid tokens to bubble through the preprocessor so that they have chance to be skipped.
Review URL: https://codereview.appspot.com/6356045
git-svn-id: http://angleproject.googlecode.com/svn/trunk@1169 736b8ea6-26fd-11df-bfd4-992fa37f6226
diff --git a/src/compiler/preprocessor/new/DirectiveParser.cpp b/src/compiler/preprocessor/new/DirectiveParser.cpp
index b7a03a1..585a90e 100644
--- a/src/compiler/preprocessor/new/DirectiveParser.cpp
+++ b/src/compiler/preprocessor/new/DirectiveParser.cpp
@@ -221,7 +221,7 @@
{
mTokenizer->lex(token);
- if (token->type == '#')
+ if (token->type == Token::PP_HASH)
{
parseDirective(token);
}
@@ -242,7 +242,7 @@
void DirectiveParser::parseDirective(Token* token)
{
- assert(token->type == '#');
+ assert(token->type == Token::PP_HASH);
mTokenizer->lex(token);
DirectiveType directive = getDirective(token);
diff --git a/src/compiler/preprocessor/new/Preprocessor.cpp b/src/compiler/preprocessor/new/Preprocessor.cpp
index ed6d1d5..21901fa 100644
--- a/src/compiler/preprocessor/new/Preprocessor.cpp
+++ b/src/compiler/preprocessor/new/Preprocessor.cpp
@@ -6,8 +6,10 @@
#include "Preprocessor.h"
+#include <cassert>
#include <sstream>
+#include "Diagnostics.h"
#include "DirectiveParser.h"
#include "Macro.h"
#include "MacroExpander.h"
@@ -19,16 +21,18 @@
struct PreprocessorImpl
{
+ Diagnostics* diagnostics;
MacroSet macroSet;
Tokenizer tokenizer;
DirectiveParser directiveParser;
MacroExpander macroExpander;
- PreprocessorImpl(Diagnostics* diagnostics,
+ PreprocessorImpl(Diagnostics* diag,
DirectiveHandler* directiveHandler) :
- tokenizer(diagnostics),
- directiveParser(&tokenizer, ¯oSet, diagnostics, directiveHandler),
- macroExpander(&directiveParser, ¯oSet, diagnostics)
+ diagnostics(diag),
+ tokenizer(diag),
+ directiveParser(&tokenizer, ¯oSet, diag, directiveHandler),
+ macroExpander(&directiveParser, ¯oSet, diag)
{
}
};
@@ -79,7 +83,31 @@
void Preprocessor::lex(Token* token)
{
- mImpl->macroExpander.lex(token);
+ bool validToken = false;
+ while (!validToken)
+ {
+ mImpl->macroExpander.lex(token);
+ switch (token->type)
+ {
+ // We should not be returning internal preprocessing tokens.
+ // Convert preprocessing tokens to compiler tokens or report
+ // diagnostics.
+ case Token::PP_HASH:
+ assert(false);
+ break;
+ case Token::PP_NUMBER:
+ mImpl->diagnostics->report(Diagnostics::INVALID_NUMBER,
+ token->location, token->value);
+ break;
+ case Token::PP_OTHER:
+ mImpl->diagnostics->report(Diagnostics::INVALID_CHARACTER,
+ token->location, token->value);
+ break;
+ default:
+ validToken = true;
+ break;
+ }
+ }
}
} // namespace pp
diff --git a/src/compiler/preprocessor/new/Token.h b/src/compiler/preprocessor/new/Token.h
index 55b708e..5231021 100644
--- a/src/compiler/preprocessor/new/Token.h
+++ b/src/compiler/preprocessor/new/Token.h
@@ -46,7 +46,14 @@
OP_RIGHT_ASSIGN,
OP_AND_ASSIGN,
OP_XOR_ASSIGN,
- OP_OR_ASSIGN
+ OP_OR_ASSIGN,
+
+ // Preprocessing token types.
+ // These types are used by the preprocessor internally.
+ // Preprocessor clients must not depend or check for them.
+ PP_HASH,
+ PP_NUMBER,
+ PP_OTHER
};
enum Flags
{
diff --git a/src/compiler/preprocessor/new/Tokenizer.cpp b/src/compiler/preprocessor/new/Tokenizer.cpp
index f5d3976..7a22524 100644
--- a/src/compiler/preprocessor/new/Tokenizer.cpp
+++ b/src/compiler/preprocessor/new/Tokenizer.cpp
@@ -898,14 +898,8 @@
YY_RULE_SETUP
{
// # is only valid at start of line for preprocessor directives.
- if (yyextra->lineStart) {
- yylval->assign(1, yytext[0]);
- return yytext[0];
- } else {
- yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
- pp::SourceLocation(yyfileno, yylineno),
- std::string(yytext, yyleng));
- }
+ yylval->assign(1, yytext[0]);
+ return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
}
YY_BREAK
case 8:
@@ -934,9 +928,8 @@
case 11:
YY_RULE_SETUP
{
- yyextra->diagnostics->report(pp::Diagnostics::INVALID_NUMBER,
- pp::SourceLocation(yyfileno, yylineno),
- std::string(yytext, yyleng));
+ yylval->assign(yytext, yyleng);
+ return pp::Token::PP_NUMBER;
}
YY_BREAK
case 12:
@@ -1109,9 +1102,8 @@
case 36:
YY_RULE_SETUP
{
- yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
- pp::SourceLocation(yyfileno, yylineno),
- std::string(yytext, yyleng));
+ yylval->assign(1, yytext[0]);
+ return pp::Token::PP_OTHER;
}
YY_BREAK
case YY_STATE_EOF(INITIAL):
diff --git a/src/compiler/preprocessor/new/Tokenizer.l b/src/compiler/preprocessor/new/Tokenizer.l
index 5b1c953..49c4c79 100644
--- a/src/compiler/preprocessor/new/Tokenizer.l
+++ b/src/compiler/preprocessor/new/Tokenizer.l
@@ -100,14 +100,8 @@
# {
// # is only valid at start of line for preprocessor directives.
- if (yyextra->lineStart) {
- yylval->assign(1, yytext[0]);
- return yytext[0];
- } else {
- yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
- pp::SourceLocation(yyfileno, yylineno),
- std::string(yytext, yyleng));
- }
+ yylval->assign(1, yytext[0]);
+ return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
}
{IDENTIFIER} {
@@ -128,9 +122,8 @@
/* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
/* Rule to catch all invalid integers and floats. */
({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
- yyextra->diagnostics->report(pp::Diagnostics::INVALID_NUMBER,
- pp::SourceLocation(yyfileno, yylineno),
- std::string(yytext, yyleng));
+ yylval->assign(yytext, yyleng);
+ return pp::Token::PP_NUMBER;
}
"++" {
@@ -232,9 +225,8 @@
}
. {
- yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
- pp::SourceLocation(yyfileno, yylineno),
- std::string(yytext, yyleng));
+ yylval->assign(1, yytext[0]);
+ return pp::Token::PP_OTHER;
}
<*><<EOF>> {
