Merge remote-tracking branch 'aosp/upstream-master' into update-shaderc
Includes:
be8e087 Travis: download and install pip
4f36a64 Force binary mode when writing a binary to stdout
87a8420 Adjust configuration to link SPIRV-Tools into Glslang
Change-Id: I3913a24661c82c3c601474af4b3e4d9dccafd1ec
Testing: checkbuild.py on Linux; unit tests on Windows
diff --git a/.travis.yml b/.travis.yml
index f0b2513..c1aaee6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -56,11 +56,12 @@
git clone --depth=1 https://github.com/taka-no-me/android-cmake.git $HOME/android-cmake;
export TOOLCHAIN_PATH=$HOME/android-cmake/android.toolchain.cmake;
fi
+ - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+ curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py;
+ sudo python /tmp/get-pip.py;
+ fi
install:
- - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
- sudo easy_install pip;
- fi
- pip install --user nose
- pip install --user cpp-coveralls
diff --git a/glslc/src/file_compiler.cc b/glslc/src/file_compiler.cc
index b115d07..839cc6e 100644
--- a/glslc/src/file_compiler.cc
+++ b/glslc/src/file_compiler.cc
@@ -214,7 +214,13 @@
case SpirvBinaryEmissionFormat::Unspecified:
case SpirvBinaryEmissionFormat::Binary:
// The output format is unspecified or specified as binary output.
+ // On Windows, the output stream must be set to binary mode. By
+ // default the standard output stream is set to text mode, which
+ // translates newlines (\n) to carriage-return newline pairs
+ // (\r\n).
+ if (out == &std::cout) shaderc_util::FlushAndSetBinaryModeOnStdout();
out->write(compilation_output.data(), compilation_output.size());
+ if (out == &std::cout) shaderc_util::FlushAndSetTextModeOnStdout();
break;
case SpirvBinaryEmissionFormat::Numbers:
// The output format is specified to be a list of hex numbers, the
diff --git a/glslc/test/expect.py b/glslc/test/expect.py
index 366fdec..0226119 100644
--- a/glslc/test/expect.py
+++ b/glslc/test/expect.py
@@ -113,11 +113,20 @@
return False, 'Extra files generated: {}'.format(generated_files)
-class CorrectObjectFilePreamble(GlslCTest):
- """Provides methods for verifying preamble for a SPV object file."""
+class CorrectBinaryLengthAndPreamble(GlslCTest):
+ """Provides methods for verifying preamble for a SPIR-V binary."""
- def verify_object_file_preamble(self, filename, spv_version = 0x10000):
- """Checks that the given SPIR-V binary file has correct preamble."""
+ def verify_binary_length_and_header(self, binary, spv_version = 0x10000):
+ """Checks that the given SPIR-V binary has valid length and header.
+
+ Returns:
+ False, error string if anything is invalid
+ True, '' otherwise
+ Args:
+ binary: a bytes object containing the SPIR-V binary
+ spv_version: target SPIR-V version number, with same encoding
+ as the version word in a SPIR-V header.
+ """
def read_word(binary, index, little_endian):
"""Reads the index-th word from the given binary file."""
@@ -127,7 +136,7 @@
return reduce(lambda w, b: (w << 8) | ord(b), word, 0)
def check_endianness(binary):
- """Checks the endianness of the given SPIR-V binary file.
+ """Checks the endianness of the given SPIR-V binary.
Returns:
True if it's little endian, False if it's big endian.
@@ -141,6 +150,45 @@
return False
return None
+ num_bytes = len(binary)
+ if num_bytes % 4 != 0:
+ return False, ('Incorrect SPV binary: size should be a multiple'
+ ' of words')
+ if num_bytes < 20:
+ return False, 'Incorrect SPV binary: size less than 5 words'
+
+ preamble = binary[0:19]
+ little_endian = check_endianness(preamble)
+ # SPIR-V module magic number
+ if little_endian is None:
+ return False, 'Incorrect SPV binary: wrong magic number'
+
+ # SPIR-V version number
+ version = read_word(preamble, 1, little_endian)
+ # TODO(dneto): Recent Glslang uses version word 0 for opengl_compat
+ # profile
+
+ if version != spv_version and version != 0:
+ return False, 'Incorrect SPV binary: wrong version number'
+ # Shaderc-over-Glslang (0x000d....) or
+ # SPIRV-Tools (0x0007....) generator number
+ if read_word(preamble, 2, little_endian) != 0x000d0006 and \
+ read_word(preamble, 2, little_endian) != 0x00070000:
+ return False, ('Incorrect SPV binary: wrong generator magic '
+ 'number')
+ # reserved for instruction schema
+ if read_word(preamble, 4, little_endian) != 0:
+ return False, 'Incorrect SPV binary: the 5th byte should be 0'
+
+ return True, ''
+
+
+class CorrectObjectFilePreamble(CorrectBinaryLengthAndPreamble):
+ """Provides methods for verifying preamble for a SPV object file."""
+
+ def verify_object_file_preamble(self, filename, spv_version = 0x10000):
+ """Checks that the given SPIR-V binary file has correct preamble."""
+
success, message = verify_file_non_empty(filename)
if not success:
return False, message
@@ -148,36 +196,11 @@
with open(filename, 'rb') as object_file:
object_file.seek(0, os.SEEK_END)
num_bytes = object_file.tell()
- if num_bytes % 4 != 0:
- return False, ('Incorrect SPV binary: size should be a multiple'
- ' of words')
- if num_bytes < 20:
- return False, 'Incorrect SPV binary: size less than 5 words'
object_file.seek(0)
- preamble = bytes(object_file.read(20))
- little_endian = check_endianness(preamble)
- # SPIR-V module magic number
- if little_endian is None:
- return False, 'Incorrect SPV binary: wrong magic number'
-
- # SPIR-V version number
- version = read_word(preamble, 1, little_endian)
- # TODO(dneto): Recent Glslang uses version word 0 for opengl_compat
- # profile
-
- if version != spv_version and version != 0:
- return False, 'Incorrect SPV binary: wrong version number'
- # Shaderc-over-Glslang (0x000d....) or
- # SPIRV-Tools (0x0007....) generator number
- if read_word(preamble, 2, little_endian) != 0x000d0006 and \
- read_word(preamble, 2, little_endian) != 0x00070000:
- return False, ('Incorrect SPV binary: wrong generator magic '
- 'number')
- # reserved for instruction schema
- if read_word(preamble, 4, little_endian) != 0:
- return False, 'Incorrect SPV binary: the 5th byte should be 0'
+ binary = bytes(object_file.read())
+ return self.verify_binary_length_and_header(binary, spv_version)
return True, ''
diff --git a/glslc/test/option_dash_o.py b/glslc/test/option_dash_o.py
index 1a32c12..8fdb89b 100644
--- a/glslc/test/option_dash_o.py
+++ b/glslc/test/option_dash_o.py
@@ -63,3 +63,35 @@
glslc_args = ['-o']
expected_error = ['glslc: error: argument to \'-o\' is missing ' +
'(expected 1 value)\n']
+
+
+@inside_glslc_testsuite('OptionDashO')
+class OutputFileBinaryAvoidsCRLFTranslation(expect.ReturnCodeIsZero,
+ expect.NoOutputOnStderr,
+ expect.NoGeneratedFiles,
+ expect.CorrectBinaryLengthAndPreamble):
+ """Tests that the -o flag emits a binary file without CR/LF translation.
+ """
+
+ # A shader whose compiled output has three bytes that are newlines.
+ # If the output stream converts the newlines to CR/LF, then we end up
+ # with a file that is 4k + 3 bytes long. That will be caught by the
+ # object file checks.
+ SHADER_WITH_THREE_NEWLINES_IN_BINARY = """#version 450
+ layout(location = 0) out uint ovar;
+ void main() { ovar = 10; }
+ """
+
+ shader = FileShader(SHADER_WITH_THREE_NEWLINES_IN_BINARY, '.vert')
+ glslc_args = [shader, '-o', '-']
+
+ def check_stdout_binary(self, status):
+ binary = status.stdout
+ newlines = [x for x in binary if x == '\n']
+ num_newlines = len(newlines)
+ if num_newlines % 4 == 0:
+ return False, "Bad test. Need nontrivial number of newlines"
+ if num_newlines != 3:
+ return False, ("Update this test. Expected 3 newlines in the "
+ "binary, but found {}").format(num_newlines)
+ return self.verify_binary_length_and_header(status.stdout)
diff --git a/libshaderc_util/include/libshaderc_util/io.h b/libshaderc_util/include/libshaderc_util/io.h
index eb67091..b9116c1 100644
--- a/libshaderc_util/include/libshaderc_util/io.h
+++ b/libshaderc_util/include/libshaderc_util/io.h
@@ -57,6 +57,13 @@
// is "-", writes to std::cout.
bool WriteFile(std::ostream* output_stream, const string_piece& output_data);
+// Flush the standard output stream and set it to binary mode. Subsequent
+// output will not translate newlines to carriage-return newline pairs.
+void FlushAndSetBinaryModeOnStdout();
+// Flush the standard output stream and set it to text mode. Subsequent
+// output will translate newlines to carriage-return newline pairs.
+void FlushAndSetTextModeOnStdout();
+
} // namespace shaderc_util
#endif // LIBSHADERC_UTIL_IO_H_
diff --git a/libshaderc_util/src/io.cc b/libshaderc_util/src/io.cc
index 72a4055..42ae89d 100644
--- a/libshaderc_util/src/io.cc
+++ b/libshaderc_util/src/io.cc
@@ -16,7 +16,15 @@
#include "libshaderc_util/universal_unistd.h"
+#if _WIN32
+// Need _fileno from stdio.h
+// Need _O_BINARY and _O_TEXT from fcntl.h
+#include <fcntl.h>
+#include <stdio.h>
+#endif
+
#include <errno.h>
+#include <cstdio>
#include <cstring>
#include <fstream>
#include <iostream>
@@ -120,4 +128,18 @@
return true;
}
+void FlushAndSetBinaryModeOnStdout() {
+ std::fflush(stdout);
+#if _WIN32
+ _setmode(_fileno(stdout), _O_BINARY);
+#endif
+}
+
+void FlushAndSetTextModeOnStdout() {
+ std::fflush(stdout);
+#if _WIN32
+ _setmode(_fileno(stdout), _O_TEXT);
+#endif
+}
+
} // namespace shaderc_util
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index 373d5b4..b2f677b 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -32,6 +32,19 @@
add_subdirectory(${SHADERC_SPIRV_HEADERS_DIR} spirv-headers)
endif()
+# Check SPIRV-Tools before glslang so that it is linked into glslang.
+# we control optimizations via glslang API calls directly.
+if (IS_DIRECTORY ${SHADERC_SPIRV_TOOLS_DIR})
+ if ("${SHADERC_SKIP_TESTS}")
+ # Also skip building tests in SPIRV-Tools.
+ set(SPIRV_SKIP_TESTS ON CACHE BOOL "Skip building SPIRV-Tools tests")
+ endif()
+ add_subdirectory(${SHADERC_SPIRV_TOOLS_DIR} spirv-tools)
+endif()
+if (NOT TARGET SPIRV-Tools)
+ message(FATAL_ERROR "SPIRV-Tools was not found - required for compilation")
+endif()
+
if (IS_DIRECTORY ${SHADERC_GLSLANG_DIR})
add_subdirectory(${SHADERC_GLSLANG_DIR} glslang)
endif()
@@ -46,19 +59,6 @@
"Platform Toolset" FORCE)
endif()
-# Check SPIRV-Tools after glslang so that it is not linked into glslang and
-# we can control optimizations.
-if (IS_DIRECTORY ${SHADERC_SPIRV_TOOLS_DIR})
- if ("${SHADERC_SKIP_TESTS}")
- # Also skip building tests in SPIRV-Tools.
- set(SPIRV_SKIP_TESTS ON CACHE BOOL "Skip building SPIRV-Tools tests")
- endif()
- add_subdirectory(${SHADERC_SPIRV_TOOLS_DIR} spirv-tools)
-endif()
-if (NOT TARGET SPIRV-Tools)
- message(FATAL_ERROR "SPIRV-Tools was not found - required for compilation")
-endif()
-
if(${SHADERC_ENABLE_TESTS})
# Configure out-of-source-directory tests for glslang.
# The glslang project uses a bash script called "runtests" to run tests.
