Token::value now contains text for all types of tokens - not just CONST_INT, CONST_FLOAT, and IDENTIFIER. This makes it easier to report diagnostics and various preprocessor directives.
git-svn-id: https://angleproject.googlecode.com/svn/trunk@1093 736b8ea6-26fd-11df-bfd4-992fa37f6226
diff --git a/src/compiler/preprocessor/new/Tokenizer.cpp b/src/compiler/preprocessor/new/Tokenizer.cpp
index 22f4152..ff51557 100644
--- a/src/compiler/preprocessor/new/Tokenizer.cpp
+++ b/src/compiler/preprocessor/new/Tokenizer.cpp
@@ -885,13 +885,6 @@
YY_RULE_SETUP
{ ++yylineno; }
YY_BREAK
-case YY_STATE_EOF(COMMENT):
-{
- yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
- pp::SourceLocation(yyfileno, yylineno), "");
- yyterminate();
-}
- YY_BREAK
case 6:
YY_RULE_SETUP
{
@@ -904,6 +897,7 @@
{
// # is only valid at start of line for preprocessor directives.
if (yyextra->lineStart) {
+ yylval->assign(1, yytext[0]);
return yytext[0];
} else {
yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
@@ -945,91 +939,157 @@
YY_BREAK
case 12:
YY_RULE_SETUP
-{ return pp::Token::OP_INC; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_INC;
+}
YY_BREAK
case 13:
YY_RULE_SETUP
-{ return pp::Token::OP_DEC; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_DEC;
+}
YY_BREAK
case 14:
YY_RULE_SETUP
-{ return pp::Token::OP_LEFT; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_LEFT;
+}
YY_BREAK
case 15:
YY_RULE_SETUP
-{ return pp::Token::OP_RIGHT; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_RIGHT;
+}
YY_BREAK
case 16:
YY_RULE_SETUP
-{ return pp::Token::OP_LE; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_LE;
+}
YY_BREAK
case 17:
YY_RULE_SETUP
-{ return pp::Token::OP_GE; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_GE;
+}
YY_BREAK
case 18:
YY_RULE_SETUP
-{ return pp::Token::OP_EQ; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_EQ;
+}
YY_BREAK
case 19:
YY_RULE_SETUP
-{ return pp::Token::OP_NE; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_NE;
+}
YY_BREAK
case 20:
YY_RULE_SETUP
-{ return pp::Token::OP_AND; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_AND;
+}
YY_BREAK
case 21:
YY_RULE_SETUP
-{ return pp::Token::OP_XOR; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_XOR;
+}
YY_BREAK
case 22:
YY_RULE_SETUP
-{ return pp::Token::OP_OR; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_OR;
+}
YY_BREAK
case 23:
YY_RULE_SETUP
-{ return pp::Token::OP_ADD_ASSIGN; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_ADD_ASSIGN;
+}
YY_BREAK
case 24:
YY_RULE_SETUP
-{ return pp::Token::OP_SUB_ASSIGN; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_SUB_ASSIGN;
+}
YY_BREAK
case 25:
YY_RULE_SETUP
-{ return pp::Token::OP_MUL_ASSIGN; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_MUL_ASSIGN;
+}
YY_BREAK
case 26:
YY_RULE_SETUP
-{ return pp::Token::OP_DIV_ASSIGN; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_DIV_ASSIGN;
+}
YY_BREAK
case 27:
YY_RULE_SETUP
-{ return pp::Token::OP_MOD_ASSIGN; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_MOD_ASSIGN;
+}
YY_BREAK
case 28:
YY_RULE_SETUP
-{ return pp::Token::OP_LEFT_ASSIGN; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_LEFT_ASSIGN;
+}
YY_BREAK
case 29:
YY_RULE_SETUP
-{ return pp::Token::OP_RIGHT_ASSIGN; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_RIGHT_ASSIGN;
+}
YY_BREAK
case 30:
YY_RULE_SETUP
-{ return pp::Token::OP_AND_ASSIGN; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_AND_ASSIGN;
+}
YY_BREAK
case 31:
YY_RULE_SETUP
-{ return pp::Token::OP_XOR_ASSIGN; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_XOR_ASSIGN;
+}
YY_BREAK
case 32:
YY_RULE_SETUP
-{ return pp::Token::OP_OR_ASSIGN; }
+{
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_OR_ASSIGN;
+}
YY_BREAK
case 33:
YY_RULE_SETUP
-{ return yytext[0]; }
+{
+ yylval->assign(1, yytext[0]);
+ return yytext[0];
+}
YY_BREAK
case 34:
YY_RULE_SETUP
@@ -1040,6 +1100,7 @@
YY_RULE_SETUP
{
++yylineno;
+ yylval->assign(1, '\n');
return '\n';
}
YY_BREAK
@@ -1052,7 +1113,17 @@
}
YY_BREAK
case YY_STATE_EOF(INITIAL):
-{ yyterminate(); }
+case YY_STATE_EOF(COMMENT):
+{
+ if (YY_START == COMMENT)
+ {
+ yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
+ pp::SourceLocation(yyfileno, yylineno),
+ "");
+ }
+ yylval->clear();
+ yyterminate();
+}
YY_BREAK
case 37:
YY_RULE_SETUP
diff --git a/src/compiler/preprocessor/new/Tokenizer.l b/src/compiler/preprocessor/new/Tokenizer.l
index ad8e0f7..57f65e9 100644
--- a/src/compiler/preprocessor/new/Tokenizer.l
+++ b/src/compiler/preprocessor/new/Tokenizer.l
@@ -91,11 +91,6 @@
<COMMENT>[^*\r\n]+
<COMMENT>"*"
<COMMENT>{NEWLINE} { ++yylineno; }
-<COMMENT><<EOF>> {
- yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
- pp::SourceLocation(yyfileno, yylineno), "");
- yyterminate();
-}
<COMMENT>"*/" {
yyextra->leadingSpace = true;
BEGIN(INITIAL);
@@ -104,6 +99,7 @@
# {
// # is only valid at start of line for preprocessor directives.
if (yyextra->lineStart) {
+ yylval->assign(1, yytext[0]);
return yytext[0];
} else {
yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
@@ -135,33 +131,101 @@
std::string(yytext, yyleng));
}
-"++" { return pp::Token::OP_INC; }
-"--" { return pp::Token::OP_DEC; }
-"<<" { return pp::Token::OP_LEFT; }
-">>" { return pp::Token::OP_RIGHT; }
-"<=" { return pp::Token::OP_LE; }
-">=" { return pp::Token::OP_GE; }
-"==" { return pp::Token::OP_EQ; }
-"!=" { return pp::Token::OP_NE; }
-"&&" { return pp::Token::OP_AND; }
-"^^" { return pp::Token::OP_XOR; }
-"||" { return pp::Token::OP_OR; }
-"+=" { return pp::Token::OP_ADD_ASSIGN; }
-"-=" { return pp::Token::OP_SUB_ASSIGN; }
-"*=" { return pp::Token::OP_MUL_ASSIGN; }
-"/=" { return pp::Token::OP_DIV_ASSIGN; }
-"%=" { return pp::Token::OP_MOD_ASSIGN; }
-"<<=" { return pp::Token::OP_LEFT_ASSIGN; }
-">>=" { return pp::Token::OP_RIGHT_ASSIGN; }
-"&=" { return pp::Token::OP_AND_ASSIGN; }
-"^=" { return pp::Token::OP_XOR_ASSIGN; }
-"|=" { return pp::Token::OP_OR_ASSIGN; }
+"++" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_INC;
+}
+"--" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_DEC;
+}
+"<<" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_LEFT;
+}
+">>" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_RIGHT;
+}
+"<=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_LE;
+}
+">=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_GE;
+}
+"==" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_EQ;
+}
+"!=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_NE;
+}
+"&&" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_AND;
+}
+"^^" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_XOR;
+}
+"||" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_OR;
+}
+"+=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_ADD_ASSIGN;
+}
+"-=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_SUB_ASSIGN;
+}
+"*=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_MUL_ASSIGN;
+}
+"/=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_DIV_ASSIGN;
+}
+"%=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_MOD_ASSIGN;
+}
+"<<=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_LEFT_ASSIGN;
+}
+">>=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_RIGHT_ASSIGN;
+}
+"&=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_AND_ASSIGN;
+}
+"^=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_XOR_ASSIGN;
+}
+"|=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_OR_ASSIGN;
+}
-{PUNCTUATOR} { return yytext[0]; }
+{PUNCTUATOR} {
+ yylval->assign(1, yytext[0]);
+ return yytext[0];
+}
+
[ \t\v\f]+ { yyextra->leadingSpace = true; }
{NEWLINE} {
++yylineno;
+ yylval->assign(1, '\n');
return '\n';
}
@@ -171,7 +235,16 @@
std::string(yytext, yyleng));
}
-<<EOF>> { yyterminate(); }
+<*><<EOF>> {
+ if (YY_START == COMMENT)
+ {
+ yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
+ pp::SourceLocation(yyfileno, yylineno),
+ "");
+ }
+ yylval->clear();
+ yyterminate();
+}
%%
diff --git a/tests/preprocessor_tests/char_test.cpp b/tests/preprocessor_tests/char_test.cpp
index 1123e0f..a966aef 100644
--- a/tests/preprocessor_tests/char_test.cpp
+++ b/tests/preprocessor_tests/char_test.cpp
@@ -77,6 +77,7 @@
else if (isPunctuator(str[0]))
{
expectedType = str[0];
+ expectedValue = str;
}
else if (isWhitespace(str[0]))
{
diff --git a/tests/preprocessor_tests/comment_test.cpp b/tests/preprocessor_tests/comment_test.cpp
index c8b2bcf..da54416 100644
--- a/tests/preprocessor_tests/comment_test.cpp
+++ b/tests/preprocessor_tests/comment_test.cpp
@@ -53,7 +53,7 @@
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
- EXPECT_STREQ("bar", token.value.c_str());
+ EXPECT_EQ("bar", token.value);
EXPECT_TRUE(token.hasLeadingSpace());
}
diff --git a/tests/preprocessor_tests/identifier_test.cpp b/tests/preprocessor_tests/identifier_test.cpp
index 0c30288..ce7761f 100644
--- a/tests/preprocessor_tests/identifier_test.cpp
+++ b/tests/preprocessor_tests/identifier_test.cpp
@@ -19,7 +19,7 @@
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
- EXPECT_STREQ(str, token.value.c_str());
+ EXPECT_EQ(str, token.value);
}
#if GTEST_HAS_COMBINE
diff --git a/tests/preprocessor_tests/location_test.cpp b/tests/preprocessor_tests/location_test.cpp
index 3b3c5a7..d3d3f58 100644
--- a/tests/preprocessor_tests/location_test.cpp
+++ b/tests/preprocessor_tests/location_test.cpp
@@ -21,7 +21,7 @@
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
- EXPECT_STREQ("foo", token.value.c_str());
+ EXPECT_EQ("foo", token.value);
EXPECT_EQ(location.file, token.location.file);
EXPECT_EQ(location.line, token.location.line);
diff --git a/tests/preprocessor_tests/number_test.cpp b/tests/preprocessor_tests/number_test.cpp
index 17df22a..b03347b 100644
--- a/tests/preprocessor_tests/number_test.cpp
+++ b/tests/preprocessor_tests/number_test.cpp
@@ -60,7 +60,7 @@
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::CONST_INT, token.type);
- EXPECT_STREQ(cstr, token.value.c_str());
+ EXPECT_EQ(str, token.value);
}
#define CLOSED_RANGE(x, y) testing::Range(x, static_cast<char>((y) + 1))
@@ -99,7 +99,7 @@
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::CONST_FLOAT, token.type);
- EXPECT_STREQ(str, token.value.c_str());
+ EXPECT_EQ(str, token.value);
}
typedef std::tr1::tuple<char, char, const char*, char> FloatScientificParams;
diff --git a/tests/preprocessor_tests/operator_test.cpp b/tests/preprocessor_tests/operator_test.cpp
index a619bb7..e9ccada 100644
--- a/tests/preprocessor_tests/operator_test.cpp
+++ b/tests/preprocessor_tests/operator_test.cpp
@@ -33,6 +33,7 @@
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(param.op, token.type);
+ EXPECT_EQ(param.str, token.value);
}
static const OperatorTestParam kOperators[] = {
diff --git a/tests/preprocessor_tests/space_test.cpp b/tests/preprocessor_tests/space_test.cpp
index 8a60eb6..6e9da36 100644
--- a/tests/preprocessor_tests/space_test.cpp
+++ b/tests/preprocessor_tests/space_test.cpp
@@ -39,7 +39,7 @@
// Identifier "foo" is returned after ignoring the whitespace characters.
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
- EXPECT_STREQ(identifier, token.value.c_str());
+ EXPECT_EQ(identifier, token.value);
// The whitespace character is however recorded with the next token.
EXPECT_TRUE(token.hasLeadingSpace());
}
@@ -78,7 +78,7 @@
preprocessor.lex(&token);
// Identifier "foo" is returned after ignoring the whitespace characters.
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
- EXPECT_STREQ(identifier, token.value.c_str());
+ EXPECT_EQ(identifier, token.value);
// The whitespace character is however recorded with the next token.
EXPECT_TRUE(token.hasLeadingSpace());
}
@@ -105,7 +105,7 @@
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
- EXPECT_STREQ("foo", token.value.c_str());
+ EXPECT_EQ("foo", token.value);
EXPECT_TRUE(token.hasLeadingSpace());
preprocessor.lex(&token);
@@ -118,6 +118,6 @@
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
- EXPECT_STREQ("bar", token.value.c_str());
+ EXPECT_EQ("bar", token.value);
EXPECT_FALSE(token.hasLeadingSpace());
}
diff --git a/tests/preprocessor_tests/token_test.cpp b/tests/preprocessor_tests/token_test.cpp
index bb3ab63..b0f1d57 100644
--- a/tests/preprocessor_tests/token_test.cpp
+++ b/tests/preprocessor_tests/token_test.cpp
@@ -14,7 +14,7 @@
EXPECT_EQ(0, token.flags);
EXPECT_EQ(0, token.location.line);
EXPECT_EQ(0, token.location.file);
- EXPECT_STREQ("", token.value.c_str());
+ EXPECT_EQ("", token.value);
}
TEST(TokenTest, Assignment)
@@ -31,7 +31,7 @@
EXPECT_EQ(0, token.flags);
EXPECT_EQ(0, token.location.line);
EXPECT_EQ(0, token.location.file);
- EXPECT_STREQ("", token.value.c_str());
+ EXPECT_EQ("", token.value);
}
TEST(TokenTest, Equals)
@@ -79,11 +79,11 @@
std::stringstream out1;
out1 << token;
EXPECT_TRUE(out1.good());
- EXPECT_STREQ("foo", out1.str().c_str());
+ EXPECT_EQ("foo", out1.str());
token.setHasLeadingSpace(true);
std::stringstream out2;
out2 << token;
EXPECT_TRUE(out2.good());
- EXPECT_STREQ(" foo", out2.str().c_str());
+ EXPECT_EQ(" foo", out2.str());
}
