Integrate built-in bisection search with runtest
Adds a --bisection-search switch to run-test. When this switch
is enabled run-test performs bisection bug search for nonchecker
failing tests attempting to find faulty method and optimization.
Adds ART_TEST_BISECTION flag support to Android.run-test.mk.
When this flag is set to true, run-test will run in bisection
search mode.
Bisection search works in no-prebuild mode only.
Test: ART_TEST_BISECTION=true m test-art-run-test
Change-Id: Id2d664a0b35fed366f50a60ce96f1ca6bd123cd4
diff --git a/test/Android.run-test.mk b/test/Android.run-test.mk
index e12fd28..f124b52 100644
--- a/test/Android.run-test.mk
+++ b/test/Android.run-test.mk
@@ -47,6 +47,14 @@
ART_TEST_WITH_STRACE := true
endif
+ifeq ($(ART_TEST_BISECTION),true)
+ # Need to keep rebuilding the test to bisection search it.
+ ART_TEST_RUN_TEST_NO_PREBUILD := true
+ ART_TEST_RUN_TEST_PREBUILD := false
+ # Bisection search writes to standard output.
+ ART_TEST_QUIET := false
+endif
+
# Helper to create individual build targets for tests. Must be called with $(eval).
# $(1): the test number
define define-build-art-run-test
@@ -644,6 +652,38 @@
TEST_ART_BROKEN_OPTIMIZING_HEAP_POISONING_RUN_TESTS :=
+# Tests incompatible with bisection bug search. Sorted by incompatibility reason.
+# 000 through 595 do not compile anything. 089 tests a build failure. 018 through 137
+# run dalvikvm more than once. 115 and 088 assume they are always compiled.
+# 055 tests performance which is degraded during bisecting.
+TEST_ART_INCOMPATIBLE_BISECTION_SEARCH_RUN_TESTS := \
+ 000-nop \
+ 134-nodex2oat-nofallback \
+ 147-stripped-dex-fallback \
+ 595-profile-saving \
+ \
+ 089-many-methods \
+ \
+ 018-stack-overflow \
+ 116-nodex2oat \
+ 117-nopatchoat \
+ 118-noimage-dex2oat \
+ 119-noimage-patchoat \
+ 126-miranda-multidex \
+ 137-cfi \
+ \
+ 115-native-bridge \
+ 088-monitor-verification \
+ \
+ 055-enum-performance
+
+ifeq ($(ART_TEST_BISECTION),true)
+ ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \
+ $(PREBUILD_TYPES),$(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
+ $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \
+ $(TEST_ART_INCOMPATIBLE_BISECTION_SEARCH_RUN_TESTS),$(ALL_ADDRESS_SIZES))
+endif
+
# Clear variables ahead of appending to them when defining tests.
$(foreach target, $(TARGET_TYPES), $(eval ART_RUN_TEST_$(call name-to-var,$(target))_RULES :=))
$(foreach target, $(TARGET_TYPES), \
@@ -762,6 +802,9 @@
ifeq ($(ART_TEST_RUN_TEST_ALWAYS_CLEAN),true)
run_test_options += --always-clean
endif
+ ifeq ($(ART_TEST_BISECTION),true)
+ run_test_options += --bisection-search
+ endif
ifeq ($(1),host)
uc_host_or_target := HOST
test_groups := ART_RUN_TEST_HOST_RULES
diff --git a/test/etc/run-test-jar b/test/etc/run-test-jar
index d12bd79..88fe661 100755
--- a/test/etc/run-test-jar
+++ b/test/etc/run-test-jar
@@ -55,6 +55,8 @@
USE_DEX2OAT_AND_PATCHOAT="y"
INSTRUCTION_SET_FEATURES=""
ARGS=""
+EXTERNAL_LOG_TAGS="n" # if y respect externally set ANDROID_LOG_TAGS.
+DRY_RUN="n" # if y prepare to run the test but don't run it.
while true; do
if [ "x$1" = "x--quiet" ]; then
@@ -233,6 +235,12 @@
fi
EXPERIMENTAL="$EXPERIMENTAL $2"
shift 2
+ elif [ "x$1" = "x--external-log-tags" ]; then
+ EXTERNAL_LOG_TAGS="y"
+ shift
+ elif [ "x$1" = "x--dry-run" ]; then
+ DRY_RUN="y"
+ shift
elif expr "x$1" : "x--" >/dev/null 2>&1; then
echo "unknown $0 option: $1" 1>&2
exit 1
@@ -491,12 +499,14 @@
# Create a script with the command. The command can get longer than the longest
# allowed adb command and there is no way to get the exit status from a adb shell
- # command.
+ # command. Dalvik cache is cleaned before running to make subsequent executions
+ # of the script follow the same runtime path.
cmdline="cd $DEX_LOCATION && \
export ANDROID_DATA=$DEX_LOCATION && \
export ANDROID_ADDITIONAL_PUBLIC_LIBRARIES=$PUBLIC_LIBS && \
export DEX_LOCATION=$DEX_LOCATION && \
export ANDROID_ROOT=$ANDROID_ROOT && \
+ rm -rf ${DEX_LOCATION}/dalvik-cache/ && \
mkdir -p ${mkdir_locations} && \
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH && \
export PATH=$ANDROID_ROOT/bin:$PATH && \
@@ -517,7 +527,9 @@
adb push $cmdfile $DEX_LOCATION/cmdline.sh > /dev/null 2>&1
fi
- adb shell sh $DEX_LOCATION/cmdline.sh
+ if [ "$DRY_RUN" != "y" ]; then
+ adb shell sh $DEX_LOCATION/cmdline.sh
+ fi
rm -f $cmdfile
else
@@ -525,10 +537,12 @@
# By default, and for prebuild dex2oat, we are interested in errors being logged. In dev mode
# we want debug messages.
- if [ "$DEV_MODE" = "y" ]; then
- export ANDROID_LOG_TAGS='*:d'
- else
- export ANDROID_LOG_TAGS='*:e'
+ if [ "$EXTERNAL_LOG_TAGS" = "n" ]; then
+ if [ "$DEV_MODE" = "y" ]; then
+ export ANDROID_LOG_TAGS='*:d'
+ else
+ export ANDROID_LOG_TAGS='*:e'
+ fi
fi
export ANDROID_DATA="$DEX_LOCATION"
@@ -582,15 +596,21 @@
# For running, we must turn off logging when dex2oat or patchoat are missing. Otherwise we use
# the same defaults as for prebuilt: everything when --dev, otherwise errors and above only.
- if [ "$DEV_MODE" = "y" ]; then
- export ANDROID_LOG_TAGS='*:d'
- elif [ "$USE_DEX2OAT_AND_PATCHOAT" = "n" ]; then
- # All tests would log the error of failing dex2oat/patchoat. Be silent here and only
- # log fatal events.
- export ANDROID_LOG_TAGS='*:s'
- else
- # We are interested in LOG(ERROR) output.
- export ANDROID_LOG_TAGS='*:e'
+ if [ "$EXTERNAL_LOG_TAGS" = "n" ]; then
+ if [ "$DEV_MODE" = "y" ]; then
+ export ANDROID_LOG_TAGS='*:d'
+ elif [ "$USE_DEX2OAT_AND_PATCHOAT" = "n" ]; then
+ # All tests would log the error of failing dex2oat/patchoat. Be silent here and only
+ # log fatal events.
+ export ANDROID_LOG_TAGS='*:s'
+ else
+ # We are interested in LOG(ERROR) output.
+ export ANDROID_LOG_TAGS='*:e'
+ fi
+ fi
+
+ if [ "$DRY_RUN" = "y" ]; then
+ exit 0
fi
if [ "$USE_GDB" = "y" ]; then
diff --git a/test/run-test b/test/run-test
index 8fb2adf..34f4906 100755
--- a/test/run-test
+++ b/test/run-test
@@ -130,6 +130,7 @@
pic_image_suffix=""
multi_image_suffix=""
android_root="/system"
+bisection_search="no"
# By default we will use optimizing.
image_args=""
image_suffix=""
@@ -347,6 +348,9 @@
shift
run_args="${run_args} --instruction-set-features $1"
shift
+ elif [ "x$1" = "x--bisection-search" ]; then
+ bisection_search="yes"
+ shift
elif expr "x$1" : "x--" >/dev/null 2>&1; then
echo "unknown $0 option: $1" 1>&2
usage="yes"
@@ -519,6 +523,21 @@
usage="yes"
fi
+if [ "$bisection_search" = "yes" -a "$prebuild_mode" = "yes" ]; then
+ err_echo "--bisection-search and --prebuild are mutually exclusive"
+ usage="yes"
+fi
+
+if [ "$bisection_search" = "yes" -a "$have_dex2oat" = "no" ]; then
+ err_echo "--bisection-search and --no-dex2oat are mutually exclusive"
+ usage="yes"
+fi
+
+if [ "$bisection_search" = "yes" -a "$have_patchoat" = "no" ]; then
+ err_echo "--bisection-search and --no-patchoat are mutually exclusive"
+ usage="yes"
+fi
+
if [ "$usage" = "no" ]; then
if [ "x$1" = "x" -o "x$1" = "x-" ]; then
test_dir=`basename "$oldwd"`
@@ -613,6 +632,7 @@
echo " the boot class path."
echo " --pic-test Compile the test code position independent."
echo " --quiet Don't print anything except failure messages"
+ echo " --bisection-search Perform bisection bug search."
) 1>&2 # Direct to stderr so usage is not printed if --quiet is set.
exit 1
fi
@@ -881,6 +901,41 @@
) 2>&${real_stderr} 1>&2
+# Attempt bisection only if the test failed.
+if [ "$bisection_search" = "yes" -a "$good" != "yes" ]; then
+ # Bisecting works by skipping different optimization passes which breaks checker assertions.
+ if [ "$run_checker" == "yes" ]; then
+ echo "${test_dir}: not bisecting, checker test." 1>&2
+ else
+ # Increase file size limit, bisection search can generate large logfiles.
+ if ! ulimit -S unlimited; then
+ err_echo "ulimit file size setting failed"
+ fi
+ echo "${test_dir}: bisecting..." 1>&2
+ cwd=`pwd`
+ maybe_device_mode=""
+ raw_cmd=""
+ if [ "$target_mode" = "yes" ]; then
+ # Produce cmdline.sh in $DEX_LOCATION. "$@" is passed as a runtime option
+ # so that cmdline.sh forwards its arguments to dalvikvm. invoke-with is set
+ # to exec in order to preserve pid when calling dalvikvm. This is required
+ # for bisection search to correctly retrieve logs from device.
+ "./${run}" $run_args --runtime-option '"$@"' --invoke-with exec --dry-run "$@" &> /dev/null
+ adb shell chmod u+x "$DEX_LOCATION/cmdline.sh"
+ maybe_device_mode="--device"
+ raw_cmd="$DEX_LOCATION/cmdline.sh"
+ else
+ raw_cmd="$cwd/${run} --external-log-tags $run_args $@"
+ fi
+ $ANDROID_BUILD_TOP/art/tools/bisection_search/bisection_search.py \
+ $maybe_device_mode \
+ --raw-cmd="$raw_cmd" \
+ --check-script="$cwd/check" \
+ --expected-output="$cwd/expected.txt" \
+ --timeout=300
+ fi
+fi
+
# Clean up test files.
if [ "$always_clean" = "yes" -o "$good" = "yes" ] && [ "$never_clean" = "no" ]; then
cd "$oldwd"
