projects/tensorflow/build.sh

#!/bin/bash -eu
# Copyright 2018 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################

# First, determine the latest Bazel we can support
BAZEL_VERSION=$(
  grep '_TF_MAX_BAZEL_VERSION =' configure.py | \
  cut -d\' -f2 | tr -d '[:space:]'
)
if [ -z ${BAZEL_VERSION} ]; then
  echo "Couldn't find a valid bazel version in configure.py script"
  exit 1
fi

# Then, install it
curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh
chmod +x ./bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh
./bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh

# Finally, check instalation before proceeding to compile
INSTALLED_VERSION=$(
  bazel version | grep 'Build label' | cut -d: -f2 | tr -d '[:space:]'
)
if [ ${INSTALLED_VERSION} != ${BAZEL_VERSION} ]; then
  echo "Couldn't install required Bazel. "
  echo "Want ${BAZEL_VERSION}. Got ${INSTALLED_VERSION}."
  exit 1
fi

# Generate the list of fuzzers we have (only the base/op name).
FUZZING_BUILD_FILE="tensorflow/core/kernels/fuzzing/BUILD"
declare -r FUZZERS=$(
  grep '^tf_ops_fuzz_target' ${FUZZING_BUILD_FILE} | cut -d'"' -f2 | head -n5
)

# Add a few more flags to make sure fuzzers build and run successfully.
# Note the c++11/libc++ flags to build using the same toolchain as the one used
# to build libFuzzingEngine.
CFLAGS="${CFLAGS} -fno-sanitize=vptr"
CXXFLAGS="${CXXFLAGS} -fno-sanitize=vptr -std=c++11 -stdlib=libc++"

# Make sure we run ./configure to detect when we are using a Bazel out of range
yes "" | ./configure

# See https://github.com/bazelbuild/bazel/issues/6697
sed '/::kM..SeedBytes/d' -i tensorflow/stream_executor/rng.cc

# Due to statically linking boringssl dependency, we have to define one extra
# flag when compiling for memory fuzzing (see the boringssl project).
if [ "$SANITIZER" = "memory" ]
then
  CFLAGS="${CFLAGS} -DOPENSSL_NO_ASM=1"
  CXXFLAGS="${CXXFLAGS} -DOPENSSL_NO_ASM=1"
fi

# All of the flags in $CFLAGS and $CXXFLAGS need to be passed to bazel too.
# Also, pass in flags to ensure static build and to help in debugging failures.
declare -r EXTRA_FLAGS="\
--config=monolithic --dynamic_mode=off \
--verbose_failures \
$(
for f in ${CFLAGS}; do
  echo "--conlyopt=${f}" "--linkopt=${f}"
done
for f in ${CXXFLAGS}; do
  echo "--cxxopt=${f}" "--linkopt=${f}"
done
)"

# We need a new bazel function to build the actual binary.
cat >> tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl << END

def cc_tf(name):
    native.cc_test(
        name = name + "_fuzz",
        deps = [
            "//tensorflow/core/kernels/fuzzing:fuzz_session",
            "//tensorflow/core/kernels/fuzzing:" + name + "_fuzz_lib",
            "//tensorflow/cc:cc_ops",
            "//tensorflow/cc:scope",
            "//tensorflow/core:core_cpu",
        ],
    )
END

# Import this function in the proper BUILD file.
cat >> ${FUZZING_BUILD_FILE} << END

load("//tensorflow/core/kernels/fuzzing:tf_ops_fuzz_target_lib.bzl", "cc_tf")

END

# And invoke it for all fuzzers.
for fuzzer in ${FUZZERS}; do
  echo cc_tf\(\"${fuzzer}\"\) >> ${FUZZING_BUILD_FILE}
done

# Since we force the environment, we expect bazel to fail during the linking of
# each fuzzer. Hence, we will do the linking manually at the end of the process.
# We just need to make sure we use the same invocation as bazel would use, so
# use --verbose_failures (in ${EXTRA_FLAGS}) to get it and then encode it in the
# following ${LINK_ARGS}.
declare -r LINK_ARGS="\
-pthread -fuse-ld=gold \
-Wl,-no-as-needed -Wl,-z,relro,-z,now \
-B/usr/local/bin -B/usr/bin -Wl,--gc-sections \
"

# This should always look as successful despite linking error mentioned above.
bazel build --jobs=2 ${EXTRA_FLAGS} -k //tensorflow/core/kernels/fuzzing:all || true

# For each fuzzer target, we only have to link it manually to get the binary.
for fuzzer in ${FUZZERS}; do
  fz=${fuzzer}_fuzz

  # Get the file with the parameters for linking or fail if it didn't exist.
  lfile=`ls -1 bazel-bin/tensorflow/core/kernels/fuzzing/${fz}*.params | head -n1`

  # Manually link everything.
  ${CXX} ${CXXFLAGS} $LIB_FUZZING_ENGINE -o ${OUT}/${fz} ${LINK_ARGS} -Wl,@${lfile}
done

# For coverage, we need one extra step, see the envoy and grpc projects.
if [ "$SANITIZER" = "coverage" ]
then
  declare -r REMAP_PATH=${OUT}/proc/self/cwd
  mkdir -p ${REMAP_PATH}
  rsync -ak ${SRC}/tensorflow/tensorflow ${REMAP_PATH}
  rsync -ak ${SRC}/tensorflow/third_party ${REMAP_PATH}

  # Also copy bazel generated files (via genrules)
  declare -r BAZEL_PREFIX=bazel-out/k8-opt
  declare -r REMAP_BAZEL_PATH=${REMAP_PATH}/${BAZEL_PREFIX}
  mkdir -p ${REMAP_BAZEL_PATH}
  rsync -ak ${SRC}/tensorflow/${BAZEL_PREFIX}/genfiles ${REMAP_BAZEL_PATH}

  # Finally copy the external archives source files
  rsync -ak ${SRC}/tensorflow/bazel-tensorflow/external ${REMAP_PATH}
fi

# Now that all is done, we just have to copy the existing corpora and
# dictionaries to have them available in the runtime environment.
# The tweaks to the filenames below are to make sure corpora/dictionary have
# similar names as the fuzzer binary.
for dict in tensorflow/core/kernels/fuzzing/dictionaries/*; do
  name=$(basename -- $dict)
  cp ${dict} ${OUT}/${name/.dict/_fuzz.dict}
done
for corpus in tensorflow/core/kernels/fuzzing/corpus/*; do
  name=$(basename -- $corpus)
  zip ${OUT}/${name}_fuzz_seed_corpus.zip ${corpus}/*
done

# Finally, make sure we don't accidentally run with stuff from the bazel cache.
rm -f bazel-*