Support the `exclude-filter` option with BazelTest
Prior to this change the list of excluded tests were passed to each
underlying test process. Although test modules were not run, this still
executed the TF sub-process and unnecessarily uploaded artifacts causing
performance issues.
This change addresses the issue by excluding test targets that shouldn't
be run.
The test runner queries Bazel to map module names to the corresponding
test target labels which are then excluded from the Bazel command-line.
Test: atest --host bazel-test-runner-tests
Bug: 273357727
Change-Id: Ibe9a24c944cebbdac642c42920c1793cea7d6ab8
diff --git a/atest/bazel/runner/src/com/android/tradefed/testtype/bazel/BazelTest.java b/atest/bazel/runner/src/com/android/tradefed/testtype/bazel/BazelTest.java
index c8078b1..3e2b303 100644
--- a/atest/bazel/runner/src/com/android/tradefed/testtype/bazel/BazelTest.java
+++ b/atest/bazel/runner/src/com/android/tradefed/testtype/bazel/BazelTest.java
@@ -39,6 +39,8 @@
import com.android.tradefed.util.proto.TestRecordProtoUtil;
import com.google.common.base.Throwables;
+import com.google.common.collect.SetMultimap;
+import com.google.common.collect.HashMultimap;
import com.google.common.io.CharStreams;
import com.google.common.io.MoreFiles;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos;
@@ -54,19 +56,20 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
import java.time.Duration;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.concurrent.Future;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import java.util.zip.ZipFile;
/** Test runner for executing Bazel tests. */
@@ -75,6 +78,16 @@
public static final String QUERY_TARGETS = "query_targets";
public static final String RUN_TESTS = "run_tests";
+ // TODO(b/275407694): Use the module_name parameter to filter tests instead of the query
+ // command.
+ public static final String TEST_QUERY_TEMPLATE =
+ "tests(...) - attr(module_name, \"(?:%s)\", tests(...))";
+
+ // Add method excludes to TF's global filters since Bazel doesn't support target-specific
+ // arguments. See https://github.com/bazelbuild/rules_go/issues/2784.
+ // TODO(b/274787592): Integrate with Bazel's test filtering to filter specific test cases.
+ public static final String GLOBAL_EXCLUDE_FILTER_TEMPLATE =
+ "--test_arg=--global-filters:exclude-filter=%s";
private static final Duration BAZEL_QUERY_TIMEOUT = Duration.ofMinutes(5);
private static final String TEST_NAME = BazelTest.class.getName();
@@ -91,6 +104,11 @@
private Path mRunTemporaryDirectory;
+ private enum ExcludeType {
+ MODULE,
+ TEST_CASE
+ };
+
@Option(
name = "bazel-test-command-timeout",
description = "Timeout for running the Bazel test.")
@@ -123,6 +141,9 @@
description = "Max idle timeout in seconds for bazel commands.")
private Duration mBazelMaxIdleTimeout = Duration.ofSeconds(5L);
+ @Option(name = "exclude-filter", description = "Test modules to exclude when running tests.")
+ private final List<String> mExcludeTargets = new ArrayList<>();
+
public BazelTest() {
this(new DefaultProcessStarter(), Paths.get(System.getProperty("java.io.tmpdir")));
}
@@ -301,8 +322,15 @@
ProcessBuilder builder = createBazelCommand(workspaceDirectory, QUERY_TARGETS);
+ Collection<String> moduleExcludes = groupExcludesByType().get(ExcludeType.MODULE);
+
builder.command().add("query");
- builder.command().add("tests(...)");
+ builder.command()
+ .add(
+ moduleExcludes.isEmpty()
+ ? "tests(...)"
+ : String.format(
+ TEST_QUERY_TEMPLATE, String.join("|", moduleExcludes)));
builder.redirectError(Redirect.appendTo(logFile.toFile()));
Process process = startAndWaitForProcess(QUERY_TARGETS, builder, BAZEL_QUERY_TIMEOUT);
@@ -330,12 +358,35 @@
.add(String.format("--build_event_binary_file=%s", bepFile.toAbsolutePath()));
builder.command().addAll(mBazelTestExtraArgs);
+
+ Collection<String> testFilters = groupExcludesByType().get(ExcludeType.TEST_CASE);
+ for (String test : testFilters) {
+ builder.command().add(String.format(GLOBAL_EXCLUDE_FILTER_TEMPLATE, test));
+ }
builder.redirectErrorStream(true);
builder.redirectOutput(Redirect.appendTo(logFile.toFile()));
return startProcess(RUN_TESTS, builder);
}
+ private SetMultimap<ExcludeType, String> groupExcludesByType() {
+ Map<ExcludeType, List<String>> groupedMap =
+ mExcludeTargets.stream()
+ .collect(
+ Collectors.groupingBy(
+ s ->
+ s.contains(" ")
+ ? ExcludeType.TEST_CASE
+ : ExcludeType.MODULE));
+
+ SetMultimap<ExcludeType, String> groupedMultiMap = HashMultimap.create();
+ for (Entry<ExcludeType, List<String>> entry : groupedMap.entrySet()) {
+ groupedMultiMap.putAll(entry.getKey(), entry.getValue());
+ }
+
+ return groupedMultiMap;
+ }
+
private Process startAndWaitForProcess(
String processTag, ProcessBuilder builder, Duration processTimeout)
throws InterruptedException, IOException {
diff --git a/atest/bazel/runner/tests/src/com/android/tradefed/testtype/bazel/BazelTestTest.java b/atest/bazel/runner/tests/src/com/android/tradefed/testtype/bazel/BazelTestTest.java
index d175b46..f39b9c8 100644
--- a/atest/bazel/runner/tests/src/com/android/tradefed/testtype/bazel/BazelTestTest.java
+++ b/atest/bazel/runner/tests/src/com/android/tradefed/testtype/bazel/BazelTestTest.java
@@ -33,15 +33,15 @@
import com.android.tradefed.invoker.InvocationContext;
import com.android.tradefed.invoker.TestInformation;
import com.android.tradefed.log.LogUtil.CLog;
-import com.android.tradefed.result.error.ErrorIdentifier;
-import com.android.tradefed.result.error.TestErrorIdentifier;
import com.android.tradefed.result.FailureDescription;
import com.android.tradefed.result.ILogSaverListener;
import com.android.tradefed.result.LogDataType;
import com.android.tradefed.result.LogFile;
+import com.android.tradefed.result.TestDescription;
+import com.android.tradefed.result.error.ErrorIdentifier;
+import com.android.tradefed.result.error.TestErrorIdentifier;
import com.android.tradefed.result.proto.FileProtoResultReporter;
import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
-import com.android.tradefed.result.TestDescription;
import com.android.tradefed.util.ZipUtil;
import com.google.common.base.Splitter;
@@ -52,10 +52,10 @@
import org.junit.Before;
import org.junit.Rule;
+import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-import org.junit.Test;
import org.mockito.ArgumentMatcher;
import org.mockito.InOrder;
@@ -72,13 +72,13 @@
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.TimeUnit;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.function.Function;
import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -310,6 +310,71 @@
}
@Test
+ public void excludeTestModule_generatesExcludeQuery() throws Exception {
+ String moduleExclude = "custom_module";
+ List<String> command = new ArrayList<>();
+ FakeProcessStarter processStarter = newFakeProcessStarter();
+ processStarter.put(
+ BazelTest.QUERY_TARGETS,
+ builder -> {
+ command.addAll(builder.command());
+ return newPassingProcessWithStdout("default_target");
+ });
+ BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
+ OptionSetter setter = new OptionSetter(bazelTest);
+ setter.setOptionValue("exclude-filter", moduleExclude);
+
+ bazelTest.run(mTestInfo, mMockListener);
+
+ assertThat(command)
+ .contains("tests(...) - attr(module_name, \"(?:custom_module)\", tests(...))");
+ }
+
+ @Test
+ public void excludeTestFunction_generatesExcludeFilter() throws Exception {
+ String functionExclude = "custom_module custom_module.customClass#customFunction";
+ List<String> command = new ArrayList<>();
+ FakeProcessStarter processStarter = newFakeProcessStarter();
+ processStarter.put(
+ BazelTest.RUN_TESTS,
+ builder -> {
+ command.addAll(builder.command());
+ return new FakeBazelTestProcess(builder, mBazelTempPath);
+ });
+ BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
+ OptionSetter setter = new OptionSetter(bazelTest);
+ setter.setOptionValue("exclude-filter", functionExclude);
+
+ bazelTest.run(mTestInfo, mMockListener);
+
+ assertThat(command)
+ .contains(
+ "--test_arg=--global-filters:exclude-filter=custom_module"
+ + " custom_module.customClass#customFunction");
+ }
+
+ @Test
+ public void excludeTestTarget_doesNotExcludeSelectedTests() throws Exception {
+ String moduleExclude = "custom_module";
+ List<String> command = new ArrayList<>();
+ FakeProcessStarter processStarter = newFakeProcessStarter();
+ processStarter.put(
+ BazelTest.RUN_TESTS,
+ builder -> {
+ command.addAll(builder.command());
+ return new FakeBazelTestProcess(builder, mBazelTempPath);
+ });
+ BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
+ OptionSetter setter = new OptionSetter(bazelTest);
+ setter.setOptionValue("exclude-filter", moduleExclude);
+ setter.setOptionValue("bazel-test-target-patterns", moduleExclude);
+
+ bazelTest.run(mTestInfo, mMockListener);
+
+ assertThat(command).contains(moduleExclude);
+ }
+
+ @Test
public void queryStdoutEmpty_abortsRun() throws Exception {
FakeProcessStarter processStarter = newFakeProcessStarter();
processStarter.put(BazelTest.QUERY_TARGETS, newPassingProcessWithStdout(""));
