| #!/bin/bash -eu |
| # Copyright 2018 Google Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| ################################################################################ |
| |
| # First, determine the latest Bazel we can support |
| BAZEL_VERSION=$( |
| grep '_TF_MAX_BAZEL_VERSION =' configure.py | \ |
| cut -d\' -f2 | tr -d '[:space:]' |
| ) |
| if [ -z ${BAZEL_VERSION} ]; then |
| echo "Couldn't find a valid bazel version in configure.py script" |
| exit 1 |
| fi |
| |
| # Then, install it |
| curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh |
| chmod +x ./bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh |
| ./bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh |
| |
| # Finally, check instalation before proceeding to compile |
| INSTALLED_VERSION=$( |
| bazel version | grep 'Build label' | cut -d: -f2 | tr -d '[:space:]' |
| ) |
| if [ ${INSTALLED_VERSION} != ${BAZEL_VERSION} ]; then |
| echo "Couldn't install required Bazel. " |
| echo "Want ${BAZEL_VERSION}. Got ${INSTALLED_VERSION}." |
| exit 1 |
| fi |
| |
| # Generate the list of fuzzers we have (only the base/op name). |
| FUZZING_BUILD_FILE="tensorflow/core/kernels/fuzzing/BUILD" |
| declare -r FUZZERS=$( |
| grep '^tf_ops_fuzz_target' ${FUZZING_BUILD_FILE} | cut -d'"' -f2 | head -n5 |
| ) |
| |
| # Add a few more flags to make sure fuzzers build and run successfully. |
| # Note the c++11/libc++ flags to build using the same toolchain as the one used |
| # to build libFuzzingEngine. |
| CFLAGS="${CFLAGS} -fno-sanitize=vptr" |
| CXXFLAGS="${CXXFLAGS} -fno-sanitize=vptr -std=c++11 -stdlib=libc++" |
| |
| # Make sure we run ./configure to detect when we are using a Bazel out of range |
| yes "" | ./configure |
| |
| # See https://github.com/bazelbuild/bazel/issues/6697 |
| sed '/::kM..SeedBytes/d' -i tensorflow/stream_executor/rng.cc |
| |
| # Due to statically linking boringssl dependency, we have to define one extra |
| # flag when compiling for memory fuzzing (see the boringssl project). |
| if [ "$SANITIZER" = "memory" ] |
| then |
| CFLAGS="${CFLAGS} -DOPENSSL_NO_ASM=1" |
| CXXFLAGS="${CXXFLAGS} -DOPENSSL_NO_ASM=1" |
| fi |
| |
| # All of the flags in $CFLAGS and $CXXFLAGS need to be passed to bazel too. |
| # Also, pass in flags to ensure static build and to help in debugging failures. |
| declare -r EXTRA_FLAGS="\ |
| --config=monolithic --dynamic_mode=off \ |
| --verbose_failures \ |
| $( |
| for f in ${CFLAGS}; do |
| echo "--conlyopt=${f}" "--linkopt=${f}" |
| done |
| for f in ${CXXFLAGS}; do |
| echo "--cxxopt=${f}" "--linkopt=${f}" |
| done |
| )" |
| |
| # We need a new bazel function to build the actual binary. |
| cat >> tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl << END |
| |
| def cc_tf(name): |
| native.cc_test( |
| name = name + "_fuzz", |
| deps = [ |
| "//tensorflow/core/kernels/fuzzing:fuzz_session", |
| "//tensorflow/core/kernels/fuzzing:" + name + "_fuzz_lib", |
| "//tensorflow/cc:cc_ops", |
| "//tensorflow/cc:scope", |
| "//tensorflow/core:core_cpu", |
| ], |
| ) |
| END |
| |
| # Import this function in the proper BUILD file. |
| cat >> ${FUZZING_BUILD_FILE} << END |
| |
| load("//tensorflow/core/kernels/fuzzing:tf_ops_fuzz_target_lib.bzl", "cc_tf") |
| |
| END |
| |
| # And invoke it for all fuzzers. |
| for fuzzer in ${FUZZERS}; do |
| echo cc_tf\(\"${fuzzer}\"\) >> ${FUZZING_BUILD_FILE} |
| done |
| |
| # Since we force the environment, we expect bazel to fail during the linking of |
| # each fuzzer. Hence, we will do the linking manually at the end of the process. |
| # We just need to make sure we use the same invocation as bazel would use, so |
| # use --verbose_failures (in ${EXTRA_FLAGS}) to get it and then encode it in the |
| # following ${LINK_ARGS}. |
| declare -r LINK_ARGS="\ |
| -pthread -fuse-ld=gold \ |
| -Wl,-no-as-needed -Wl,-z,relro,-z,now \ |
| -B/usr/local/bin -B/usr/bin -Wl,--gc-sections \ |
| " |
| |
| # This should always look as successful despite linking error mentioned above. |
| bazel build --jobs=2 ${EXTRA_FLAGS} -k //tensorflow/core/kernels/fuzzing:all || true |
| |
| # For each fuzzer target, we only have to link it manually to get the binary. |
| for fuzzer in ${FUZZERS}; do |
| fz=${fuzzer}_fuzz |
| |
| # Get the file with the parameters for linking or fail if it didn't exist. |
| lfile=`ls -1 bazel-bin/tensorflow/core/kernels/fuzzing/${fz}*.params | head -n1` |
| |
| # Manually link everything. |
| ${CXX} ${CXXFLAGS} $LIB_FUZZING_ENGINE -o ${OUT}/${fz} ${LINK_ARGS} -Wl,@${lfile} |
| done |
| |
| # For coverage, we need one extra step, see the envoy and grpc projects. |
| if [ "$SANITIZER" = "coverage" ] |
| then |
| declare -r REMAP_PATH=${OUT}/proc/self/cwd |
| mkdir -p ${REMAP_PATH} |
| rsync -ak ${SRC}/tensorflow/tensorflow ${REMAP_PATH} |
| rsync -ak ${SRC}/tensorflow/third_party ${REMAP_PATH} |
| |
| # Also copy bazel generated files (via genrules) |
| declare -r BAZEL_PREFIX=bazel-out/k8-opt |
| declare -r REMAP_BAZEL_PATH=${REMAP_PATH}/${BAZEL_PREFIX} |
| mkdir -p ${REMAP_BAZEL_PATH} |
| rsync -ak ${SRC}/tensorflow/${BAZEL_PREFIX}/genfiles ${REMAP_BAZEL_PATH} |
| |
| # Finally copy the external archives source files |
| rsync -ak ${SRC}/tensorflow/bazel-tensorflow/external ${REMAP_PATH} |
| fi |
| |
| # Now that all is done, we just have to copy the existing corpora and |
| # dictionaries to have them available in the runtime environment. |
| # The tweaks to the filenames below are to make sure corpora/dictionary have |
| # similar names as the fuzzer binary. |
| for dict in tensorflow/core/kernels/fuzzing/dictionaries/*; do |
| name=$(basename -- $dict) |
| cp ${dict} ${OUT}/${name/.dict/_fuzz.dict} |
| done |
| for corpus in tensorflow/core/kernels/fuzzing/corpus/*; do |
| name=$(basename -- $corpus) |
| zip ${OUT}/${name}_fuzz_seed_corpus.zip ${corpus}/* |
| done |
| |
| # Finally, make sure we don't accidentally run with stuff from the bazel cache. |
| rm -f bazel-* |