infra/base-images/base-runner/test_all

#!/bin/bash -u
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################

# Percentage threshold that needs to be reached for marking a build as failed.
ALLOWED_BROKEN_TARGETS_PERCENTAGE=${ALLOWED_BROKEN_TARGETS_PERCENTAGE:-10}

# Test all fuzz targets in the $OUT/ dir.
TOTAL_TARGETS_COUNT=0

# Number of CPUs available, this is needed for running tests in parallel.
NPROC=$(nproc)

# Directories where bad build check results will be written to.
VALID_TARGETS_DIR="/tmp/valid_fuzz_targets"
BROKEN_TARGETS_DIR="/tmp/broken_fuzz_targets"
rm -rf $VALID_TARGETS_DIR
rm -rf $BROKEN_TARGETS_DIR
mkdir $VALID_TARGETS_DIR
mkdir $BROKEN_TARGETS_DIR

# Move the directory the fuzzer is located in to somewhere that doesn't exist
# on the builder to make it more likely that hardcoding /out fails here (since
# it will fail on ClusterFuzz).
TMP_FUZZER_DIR=/tmp/not-out
rm -rf $TMP_FUZZER_DIR
mkdir $TMP_FUZZER_DIR
# Move contents of $OUT/ into $TMP_FUZZER_DIR. We can't move the directory
# itself because it is a mount.
mv $OUT/* $TMP_FUZZER_DIR
INITIAL_OUT=$OUT
export OUT=$TMP_FUZZER_DIR


# Main loop that iterates through all fuzz targets and runs the check.
for FUZZER_BINARY in $(find $TMP_FUZZER_DIR -maxdepth 1 -executable -type f); do
  if file "$FUZZER_BINARY" | grep -v ELF > /dev/null 2>&1; then
    continue
  fi

  # Continue if not a fuzz target.
  if [[ $FUZZING_ENGINE != "none" ]]; then
    grep "LLVMFuzzerTestOneInput" $FUZZER_BINARY > /dev/null 2>&1 || continue
  fi

  FUZZER=$(basename $FUZZER_BINARY)
  if [[ "$FUZZER" == afl-* ]]; then
    continue
  fi
  if [[ "$FUZZER" == honggfuzz ]]; then
    continue
  fi

  echo "INFO: performing bad build checks for $FUZZER_BINARY."

  LOG_PATH_FOR_BROKEN_TARGET="${BROKEN_TARGETS_DIR}/${FUZZER}"

  # Launch bad build check process in the background. Ignore the exit codes, as
  # we check the percentage of broken fuzz targets after running all the checks.
  bad_build_check $FUZZER_BINARY &> $LOG_PATH_FOR_BROKEN_TARGET &

  # Count total number of fuzz targets being tested.
  TOTAL_TARGETS_COUNT=$[$TOTAL_TARGETS_COUNT+1]

  # Do not spawn more processes than the number of CPUs available.
  n_child_proc=$(jobs -rp | wc -l)
  while [ "$n_child_proc" -eq "$NPROC" ]; do
    sleep 4
    n_child_proc=$(jobs -rp | wc -l)
  done
done

# Wait for background processes to finish.
wait

# Restore OUT
export OUT=$INITIAL_OUT
mv $TMP_FUZZER_DIR/* $OUT

# Sanity check in case there are no fuzz targets in the $OUT/ dir.
if [ "$TOTAL_TARGETS_COUNT" -eq "0" ]; then
  echo "ERROR: no fuzzers found in $OUT/"
  ls -al $OUT
  exit 1
fi

# An empty log file indicated that corresponding fuzz target is not broken.
find $BROKEN_TARGETS_DIR -empty -exec mv {} $VALID_TARGETS_DIR \;

# Calculate number of valid and broken fuzz targets.
VALID_TARGETS_COUNT=$(ls $VALID_TARGETS_DIR | wc -l)
BROKEN_TARGETS_COUNT=$(ls $BROKEN_TARGETS_DIR | wc -l)

# Sanity check to make sure that bad build check doesn't skip any fuzz target.
if [ "$TOTAL_TARGETS_COUNT" -ne "$[$VALID_TARGETS_COUNT+$BROKEN_TARGETS_COUNT]" ]; then
  echo "ERROR: bad_build_check seems to have a bug, total number of fuzz" \
       "does not match number of fuzz targets tested."
  echo "Total fuzz targets ($TOTAL_TARGETS_COUNT):"
  ls -al $OUT
  echo "Valid fuzz targets ($VALID_TARGETS_COUNT):"
  ls -al $VALID_TARGETS_DIR
  echo "Total fuzz targets ($BROKEN_TARGETS_COUNT):"
  ls -al $BROKEN_TARGETS_DIR
  exit 1
fi

# Build info about all broken fuzz targets (if any).
if [ "$BROKEN_TARGETS_COUNT" -gt "0" ]; then
  echo "Broken fuzz targets ($BROKEN_TARGETS_COUNT):"
  for target in $(ls $BROKEN_TARGETS_DIR); do
    echo "${target}:"
    cat ${BROKEN_TARGETS_DIR}/${target}
  done
fi

# Calculate the percentage of broken fuzz targets and make the finel decision.
BROKEN_TARGETS_PERCENTAGE=$[$BROKEN_TARGETS_COUNT*100/$TOTAL_TARGETS_COUNT]


if [ "$BROKEN_TARGETS_PERCENTAGE" -gt "$ALLOWED_BROKEN_TARGETS_PERCENTAGE" ]; then
  echo "ERROR: $BROKEN_TARGETS_PERCENTAGE% of fuzz targets seem to be broken." \
       "See the list above for a detailed information."

  # TODO: figure out how to not fail the "special" cases handled below. Those
  # are from "example" and "c-ares" projects and are too small targets to pass.
  if [ "$(ls $OUT/do_stuff_fuzzer $OUT/ares_*_fuzzer $OUT/checksum_fuzzer $OUT/xmltest $OUT/fuzz_compression_sas_rle 2>/dev/null | wc -l)" -gt "0" ]; then
    exit 0
  fi

  exit 1
else
  echo "$TOTAL_TARGETS_COUNT fuzzers total, $BROKEN_TARGETS_COUNT seem to be" \
       "broken ($BROKEN_TARGETS_PERCENTAGE%)."
  exit 0
fi