Add full support for line continuation in the preprocessor
Re-landing earlier change with constant signedness fixed (was causing
build issues on Linux).
Line continuation in ESSL 3.00 needs to be processed before tokenization,
since tokens can span the line continuation. On the other hand, ANGLE's
tokenizer keeps track of line numbers, and whenever a line continuation
appears the line number still needs to be incremented by one, just like
on a regular newline.
That's why line continuation is now implemented as follows: when the
shader strings are concatenated in Input, they are also checked for line
continuation. Whenever line continuation is encountered, the string
is cut before that point. When the tokenizer asks for more input, the
string starting from the character after line continuation is passed
to it, and the line number is incremented from Input. This way the
tokenizer can parse tokens that span multiple lines - it never sees the
line continuation - but still keeps track of the line number correctly.
Relevant spec is in ESSL 3.00 section 3.2 "Source strings".
Support for line continuation also applies to ESSL 1.00. ESSL 3.00
spec section 1.5 says that line continuation support is mandated when
an ESSL 1.00 shader is used with the OpenGL ES 3.0 API, and is optional
when ESSL 1.00 is used with the OpenGL ES 2.0 API.
TEST=dEQP-GLES3.functional.shaders.preprocessor.line_continuation.*
(all pass),
angle_unittests
BUG=angleproject:1125
Change-Id: Ic086aacac53cd75bf93c0fda782416501d2f842b
Reviewed-on: https://chromium-review.googlesource.com/294200
Reviewed-by: Jamie Madill <jmadill@chromium.org>
Tested-by: Olli Etuaho <oetuaho@nvidia.com>
diff --git a/src/compiler/preprocessor/Input.cpp b/src/compiler/preprocessor/Input.cpp
index f9910a6..5541d46 100644
--- a/src/compiler/preprocessor/Input.cpp
+++ b/src/compiler/preprocessor/Input.cpp
@@ -29,13 +29,75 @@
}
}
-size_t Input::read(char *buf, size_t maxSize)
+const char *Input::skipChar()
+{
+ // This function should only be called when there is a character to skip.
+ assert(mReadLoc.cIndex < mLength[mReadLoc.sIndex]);
+ ++mReadLoc.cIndex;
+ if (mReadLoc.cIndex == mLength[mReadLoc.sIndex])
+ {
+ ++mReadLoc.sIndex;
+ mReadLoc.cIndex = 0;
+ }
+ if (mReadLoc.sIndex >= mCount)
+ {
+ return nullptr;
+ }
+ return mString[mReadLoc.sIndex] + mReadLoc.cIndex;
+}
+
+size_t Input::read(char *buf, size_t maxSize, int *lineNo)
{
size_t nRead = 0;
- while ((nRead < maxSize) && (mReadLoc.sIndex < mCount))
+ // The previous call to read might have stopped copying the string when encountering a line
+ // continuation. Check for this possibility first.
+ if (mReadLoc.sIndex < mCount && maxSize > 0)
+ {
+ const char *c = mString[mReadLoc.sIndex] + mReadLoc.cIndex;
+ if ((*c) == '\\')
+ {
+ c = skipChar();
+ if (c != nullptr && (*c) == '\n')
+ {
+ // Line continuation of backslash + newline.
+ skipChar();
+ ++(*lineNo);
+ }
+ else if (c != nullptr && (*c) == '\r')
+ {
+ // Line continuation. Could be backslash + '\r\n' or just backslash + '\r'.
+ c = skipChar();
+ if (c != nullptr && (*c) == '\n')
+ {
+ skipChar();
+ }
+ ++(*lineNo);
+ }
+ else
+ {
+ // Not line continuation, so write the skipped backslash to buf.
+ *buf = '\\';
+ ++nRead;
+ }
+ }
+ }
+
+ size_t maxRead = maxSize;
+ while ((nRead < maxRead) && (mReadLoc.sIndex < mCount))
{
size_t size = mLength[mReadLoc.sIndex] - mReadLoc.cIndex;
size = std::min(size, maxSize);
+ for (size_t i = 0; i < size; ++i)
+ {
+ // Stop if a possible line continuation is encountered.
+ // It will be processed on the next call on input, which skips it
+ // and increments line number if necessary.
+ if (*(mString[mReadLoc.sIndex] + mReadLoc.cIndex + i) == '\\')
+ {
+ size = i;
+ maxRead = nRead + size; // Stop reading right before the backslash.
+ }
+ }
std::memcpy(buf + nRead, mString[mReadLoc.sIndex] + mReadLoc.cIndex, size);
nRead += size;
mReadLoc.cIndex += size;
diff --git a/src/compiler/preprocessor/Input.h b/src/compiler/preprocessor/Input.h
index e951cb4..a1de7dd 100644
--- a/src/compiler/preprocessor/Input.h
+++ b/src/compiler/preprocessor/Input.h
@@ -33,7 +33,7 @@
return mLength[index];
}
- size_t read(char *buf, size_t maxSize);
+ size_t read(char *buf, size_t maxSize, int *lineNo);
struct Location
{
@@ -49,6 +49,10 @@
const Location &readLoc() const { return mReadLoc; }
private:
+ // Skip a character and return the next character after the one that was skipped.
+ // Return nullptr if data runs out.
+ const char *skipChar();
+
// Input.
size_t mCount;
const char * const *mString;
diff --git a/src/compiler/preprocessor/Tokenizer.cpp b/src/compiler/preprocessor/Tokenizer.cpp
index 75df434..eb6156f 100644
--- a/src/compiler/preprocessor/Tokenizer.cpp
+++ b/src/compiler/preprocessor/Tokenizer.cpp
@@ -594,7 +594,7 @@
} while(0);
#define YY_INPUT(buf, result, maxSize) \
- result = yyextra->input.read(buf, maxSize);
+ result = yyextra->input.read(buf, maxSize, &yylineno);
#define INITIAL 0
#define COMMENT 1
diff --git a/src/compiler/preprocessor/Tokenizer.l b/src/compiler/preprocessor/Tokenizer.l
index 25c938d..d316da8 100644
--- a/src/compiler/preprocessor/Tokenizer.l
+++ b/src/compiler/preprocessor/Tokenizer.l
@@ -77,7 +77,7 @@
} while(0);
#define YY_INPUT(buf, result, maxSize) \
- result = yyextra->input.read(buf, maxSize);
+ result = yyextra->input.read(buf, maxSize, &yylineno);
%}
diff --git a/src/tests/preprocessor_tests/input_test.cpp b/src/tests/preprocessor_tests/input_test.cpp
index 7f2a848..0526c4a 100644
--- a/src/tests/preprocessor_tests/input_test.cpp
+++ b/src/tests/preprocessor_tests/input_test.cpp
@@ -30,7 +30,8 @@
{
pp::Input input;
EXPECT_EQ(0u, input.count());
- EXPECT_EQ(0u, input.read(NULL, 1));
+ int lineNo = 0;
+ EXPECT_EQ(0u, input.read(NULL, 1, &lineNo));
}
TEST(InputTest, NullLength)
@@ -72,34 +73,35 @@
char buf[4] = {'\0', '\0', '\0', '\0'};
int maxSize = 1;
+ int lineNo = 0;
pp::Input input1(count, str, NULL);
- EXPECT_EQ(1u, input1.read(buf, maxSize));
+ EXPECT_EQ(1u, input1.read(buf, maxSize, &lineNo));
EXPECT_EQ('f', buf[0]);
- EXPECT_EQ(1u, input1.read(buf, maxSize));
+ EXPECT_EQ(1u, input1.read(buf, maxSize, &lineNo));
EXPECT_EQ('o', buf[0]);
- EXPECT_EQ(1u, input1.read(buf, maxSize));
+ EXPECT_EQ(1u, input1.read(buf, maxSize, &lineNo));
EXPECT_EQ('o', buf[0]);
- EXPECT_EQ(0u, input1.read(buf, maxSize));
+ EXPECT_EQ(0u, input1.read(buf, maxSize, &lineNo));
maxSize = 2;
pp::Input input2(count, str, NULL);
- EXPECT_EQ(2u, input2.read(buf, maxSize));
+ EXPECT_EQ(2u, input2.read(buf, maxSize, &lineNo));
EXPECT_STREQ("fo", buf);
- EXPECT_EQ(1u, input2.read(buf, maxSize));
+ EXPECT_EQ(1u, input2.read(buf, maxSize, &lineNo));
EXPECT_EQ('o', buf[0]);
- EXPECT_EQ(0u, input2.read(buf, maxSize));
+ EXPECT_EQ(0u, input2.read(buf, maxSize, &lineNo));
maxSize = 3;
pp::Input input3(count, str, NULL);
- EXPECT_EQ(3u, input3.read(buf, maxSize));
+ EXPECT_EQ(3u, input3.read(buf, maxSize, &lineNo));
EXPECT_STREQ("foo", buf);
- EXPECT_EQ(0u, input3.read(buf, maxSize));
+ EXPECT_EQ(0u, input3.read(buf, maxSize, &lineNo));
maxSize = 4;
pp::Input input4(count, str, NULL);
- EXPECT_EQ(3u, input4.read(buf, maxSize));
+ EXPECT_EQ(3u, input4.read(buf, maxSize, &lineNo));
EXPECT_STREQ("foo", buf);
- EXPECT_EQ(0u, input4.read(buf, maxSize));
+ EXPECT_EQ(0u, input4.read(buf, maxSize, &lineNo));
}
TEST(InputTest, ReadMultipleStrings)
@@ -109,34 +111,35 @@
char buf[4] = {'\0', '\0', '\0', '\0'};
int maxSize = 1;
+ int lineNo = 0;
pp::Input input1(count, str, NULL);
- EXPECT_EQ(1u, input1.read(buf, maxSize));
+ EXPECT_EQ(1u, input1.read(buf, maxSize, &lineNo));
EXPECT_EQ('f', buf[0]);
- EXPECT_EQ(1u, input1.read(buf, maxSize));
+ EXPECT_EQ(1u, input1.read(buf, maxSize, &lineNo));
EXPECT_EQ('o', buf[0]);
- EXPECT_EQ(1u, input1.read(buf, maxSize));
+ EXPECT_EQ(1u, input1.read(buf, maxSize, &lineNo));
EXPECT_EQ('o', buf[0]);
- EXPECT_EQ(0u, input1.read(buf, maxSize));
+ EXPECT_EQ(0u, input1.read(buf, maxSize, &lineNo));
maxSize = 2;
pp::Input input2(count, str, NULL);
- EXPECT_EQ(2u, input2.read(buf, maxSize));
+ EXPECT_EQ(2u, input2.read(buf, maxSize, &lineNo));
EXPECT_STREQ("fo", buf);
- EXPECT_EQ(1u, input2.read(buf, maxSize));
+ EXPECT_EQ(1u, input2.read(buf, maxSize, &lineNo));
EXPECT_EQ('o', buf[0]);
- EXPECT_EQ(0u, input2.read(buf, maxSize));
+ EXPECT_EQ(0u, input2.read(buf, maxSize, &lineNo));
maxSize = 3;
pp::Input input3(count, str, NULL);
- EXPECT_EQ(3u, input3.read(buf, maxSize));
+ EXPECT_EQ(3u, input3.read(buf, maxSize, &lineNo));
EXPECT_STREQ("foo", buf);
- EXPECT_EQ(0u, input3.read(buf, maxSize));
+ EXPECT_EQ(0u, input3.read(buf, maxSize, &lineNo));
maxSize = 4;
pp::Input input4(count, str, NULL);
- EXPECT_EQ(3u, input4.read(buf, maxSize));
+ EXPECT_EQ(3u, input4.read(buf, maxSize, &lineNo));
EXPECT_STREQ("foo", buf);
- EXPECT_EQ(0u, input4.read(buf, maxSize));
+ EXPECT_EQ(0u, input4.read(buf, maxSize, &lineNo));
}
TEST(InputTest, ReadStringsWithLength)
@@ -148,9 +151,28 @@
int length[] = {2, 3};
char buf[6] = {'\0', '\0', '\0', '\0', '\0', '\0'};
size_t maxSize = 5;
+ int lineNo = 0;
pp::Input input(count, str, length);
- EXPECT_EQ(maxSize, input.read(buf, maxSize));
+ EXPECT_EQ(maxSize, input.read(buf, maxSize, &lineNo));
EXPECT_STREQ("fobar", buf);
}
+TEST(InputTest, ReadStringsWithLineContinuation)
+{
+ int count = 2;
+ const char* str[] = {"foo\\", "\nba\\\r\nr"};
+ int length[] = {4, 7};
+ char buf[11] = {'\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0'};
+ size_t maxSize = 11;
+ int lineNo = 0;
+
+ pp::Input input(count, str, length);
+ EXPECT_EQ(3u, input.read(buf, maxSize, &lineNo));
+ EXPECT_EQ(0, lineNo);
+ EXPECT_EQ(2u, input.read(buf + 3, maxSize - 3, &lineNo));
+ EXPECT_EQ(1, lineNo);
+ EXPECT_EQ(1u, input.read(buf + 5, maxSize - 5, &lineNo));
+ EXPECT_EQ(2, lineNo);
+ EXPECT_STREQ("foobar", buf);
+}
