[Tradefed] Open source clustercommandscheduler
Test: Unit test passed
Bug: 144734056
Change-Id: If26f800d1b64a303a081076d8c303ed6cd13ee2a
Merged-In: If26f800d1b64a303a081076d8c303ed6cd13ee2a
diff --git a/src/com/android/tradefed/cluster/ClusterBuildInfo.java b/src/com/android/tradefed/cluster/ClusterBuildInfo.java
new file mode 100644
index 0000000..f57df27
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterBuildInfo.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.build.FolderBuildInfo;
+import com.android.tradefed.build.IBuildInfo;
+
+import java.io.File;
+
+/** A {@link IBuildInfo} class for builds piped from TFC. */
+public class ClusterBuildInfo extends FolderBuildInfo {
+
+ public ClusterBuildInfo(final File rootDir) {
+ super();
+ setRootDir(rootDir);
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterBuildProvider.java b/src/com/android/tradefed/cluster/ClusterBuildProvider.java
new file mode 100644
index 0000000..b3b67c1
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterBuildProvider.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.annotations.VisibleForTesting;
+import com.android.tradefed.build.BuildRetrievalError;
+import com.android.tradefed.build.IBuildInfo;
+import com.android.tradefed.build.IBuildProvider;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.util.ZipUtil2;
+
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+/** A {@link IBuildProvider} to download TFC test resources. */
+@OptionClass(alias = "cluster", global_namespace = false)
+public class ClusterBuildProvider implements IBuildProvider {
+
+ private static final String DEFAULT_FILE_VERSION = "0";
+
+ @Option(name = "root-dir", description = "A root directory", mandatory = true)
+ private File mRootDir;
+
+ @Option(name = "test-resource", description = "Test resources", mandatory = true)
+ private Map<String, String> mTestResources = new TreeMap<>();
+
+ @Override
+ public IBuildInfo getBuild() throws BuildRetrievalError {
+ try {
+ mRootDir.mkdirs();
+ final IBuildInfo buildInfo = new ClusterBuildInfo(mRootDir);
+ final TestResourceDownloader downloader = new TestResourceDownloader(mRootDir);
+ for (final Entry<String, String> entry : mTestResources.entrySet()) {
+ final TestResource resource = new TestResource(entry.getKey(), entry.getValue());
+ final File file = downloader.download(resource);
+ buildInfo.setFile(resource.getName(), file, DEFAULT_FILE_VERSION);
+ if (file.getName().endsWith(".zip")) {
+ // If a zip file is downloaded to a subfolder, unzip there.
+ extractZip(file, file.getParentFile());
+ }
+ }
+ return buildInfo;
+ } catch (IOException e) {
+ throw new BuildRetrievalError("failed to get test resources", e);
+ }
+ }
+
+ /** Extracts the zip to a root dir. */
+ private void extractZip(File zip, File destDir) throws IOException {
+ try (ZipFile zipFile = new ZipFile(zip)) {
+ ZipUtil2.extractZip(zipFile, destDir);
+ } catch (IOException e) {
+ throw e;
+ }
+ }
+
+ @Override
+ public void buildNotTested(IBuildInfo info) {}
+
+ @Override
+ public void cleanUp(IBuildInfo info) {
+ if (!(info instanceof ClusterBuildInfo)) {
+ throw new IllegalArgumentException("info is not an instance of ClusterBuildInfo");
+ }
+ }
+
+ @VisibleForTesting
+ File getRootDir() {
+ return mRootDir;
+ }
+
+ @VisibleForTesting
+ Map<String, String> getTestResources() {
+ return mTestResources;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterClient.java b/src/com/android/tradefed/cluster/ClusterClient.java
new file mode 100644
index 0000000..b9d7fd2
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterClient.java
@@ -0,0 +1,354 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.config.GlobalConfiguration;
+import com.android.tradefed.host.IHostOptions;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.util.StreamUtil;
+import com.android.tradefed.util.IRestApiHelper;
+import com.android.tradefed.util.RestApiHelper;
+
+import com.google.api.client.http.HttpRequestFactory;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** A {@link IClusterClient} implementation for interacting with the TFC backend. */
+public class ClusterClient implements IClusterClient {
+ private static final String DEFAULT_TFC_URL_BASE =
+ "https://tradefed-cluster.googleplex.com/_ah/api/tradefed_cluster/v1/";
+
+ /** The {@link IClusterOptions} instance used to store cluster-related settings. */
+ private IClusterOptions mClusterOptions;
+ /** The {@link IHostOptions} instance used to store host wide related settings. */
+ private IHostOptions mHostOptions;
+
+ private IRestApiHelper mApiHelper = null;
+
+ private IClusterEventUploader<ClusterCommandEvent> mCommandEventUploader = null;
+ private IClusterEventUploader<ClusterHostEvent> mHostEventUploader = null;
+
+ /**
+ * A {@link IClusterEventUploader} implementation for uploading {@link ClusterCommandEvent}s.
+ */
+ private class ClusterCommandEventUploader extends ClusterEventUploader<ClusterCommandEvent> {
+
+ private static final String REST_API_METHOD = "command_events";
+ private static final String DATA_KEY = "command_events";
+
+ @Override
+ protected void doUploadEvents(List<ClusterCommandEvent> events) throws IOException {
+ try {
+ JSONObject eventData = buildPostData(events, DATA_KEY);
+ getApiHelper().execute("POST", new String[] {REST_API_METHOD}, null, eventData);
+ } catch (JSONException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ /** A {@link IClusterEventUploader} implementation for uploading {@link ClusterHostEvent}s. */
+ private class ClusterHostEventUploader extends ClusterEventUploader<ClusterHostEvent> {
+ private static final String REST_API_METHOD = "host_events";
+ private static final String DATA_KEY = "host_events";
+
+ @Override
+ protected void doUploadEvents(List<ClusterHostEvent> events) throws IOException {
+ try {
+ JSONObject eventData = buildPostData(events, DATA_KEY);
+ getApiHelper().execute("POST", new String[] {REST_API_METHOD}, null, eventData);
+ } catch (JSONException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public List<ClusterCommand> leaseHostCommands(
+ final String clusterId,
+ final String hostname,
+ final List<ClusterDeviceInfo> deviceInfos,
+ final List<String> nextClusterIds,
+ final int maxTasksTolease)
+ throws JSONException {
+ // Make an API call to lease some work
+ final Map<String, Object> options = new HashMap<>();
+ options.put("cluster", clusterId);
+ options.put("hostname", hostname);
+ options.put("num_tasks", Integer.toString(maxTasksTolease));
+
+ JSONObject data = new JSONObject();
+ if (nextClusterIds != null && !nextClusterIds.isEmpty()) {
+ JSONArray ids = new JSONArray();
+ for (String id : nextClusterIds) {
+ ids.put(id);
+ }
+ data.put("next_cluster_ids", ids);
+ }
+ JSONArray deviceInfoJsons = new JSONArray();
+ for (ClusterDeviceInfo d : deviceInfos) {
+ deviceInfoJsons.put(d.toJSON());
+ }
+ // Add device infos in the request body. TFC will match devices based on those infos.
+ data.put("device_infos", deviceInfoJsons);
+ try {
+ // By default, execute(..) will throw an exception on HTTP error codes.
+ HttpResponse httpResponse =
+ getApiHelper()
+ .execute(
+ "POST",
+ new String[] {"tasks", "leasehosttasks"},
+ options,
+ data);
+ return parseCommandTasks(httpResponse);
+ } catch (IOException e) {
+ // May be transient. Log a warning and we'll try again later.
+ CLog.w("Failed to lease commands: %s", e);
+ return Collections.<ClusterCommand>emptyList();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public TestEnvironment getTestEnvironment(final String requestId)
+ throws IOException, JSONException {
+ final Map<String, Object> options = new HashMap<>();
+ final HttpResponse response =
+ getApiHelper()
+ .execute(
+ "GET",
+ new String[] {"requests", requestId, "test_environment"},
+ options,
+ null);
+ final String content = StreamUtil.getStringFromStream(response.getContent());
+ CLog.d(content);
+ return TestEnvironment.fromJson(new JSONObject(content));
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public List<TestResource> getTestResources(final String requestId)
+ throws IOException, JSONException {
+ final Map<String, Object> options = new HashMap<>();
+ final HttpResponse response =
+ getApiHelper()
+ .execute(
+ "GET",
+ new String[] {"requests", requestId, "test_resources"},
+ options,
+ null);
+ final String content = StreamUtil.getStringFromStream(response.getContent());
+ CLog.d(content);
+ final JSONArray jsonArray = new JSONObject(content).optJSONArray("test_resources");
+ if (jsonArray == null) {
+ return new ArrayList<>();
+ }
+ return TestResource.fromJsonArray(jsonArray);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public TestContext getTestContext(final String requestId, final String commandId)
+ throws IOException, JSONException {
+ final Map<String, Object> options = new HashMap<>();
+ final HttpResponse response =
+ getApiHelper()
+ .execute(
+ "GET",
+ new String[] {
+ "requests", requestId, "commands", commandId, "test_context"
+ },
+ options,
+ null);
+ final String content = StreamUtil.getStringFromStream(response.getContent());
+ CLog.d(content);
+ return TestContext.fromJson(new JSONObject(content));
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void updateTestContext(
+ final String requestId, final String commandId, TestContext testContext)
+ throws IOException, JSONException {
+ final Map<String, Object> options = new HashMap<>();
+ final HttpResponse response =
+ getApiHelper()
+ .execute(
+ "POST",
+ new String[] {
+ "requests", requestId, "commands", commandId, "test_context"
+ },
+ options,
+ testContext.toJson());
+ final String content = StreamUtil.getStringFromStream(response.getContent());
+ CLog.d(content);
+ }
+
+ @Override
+ public ClusterCommand.State getCommandState(String requestId, String commandId) {
+ try {
+ HttpResponse response =
+ getApiHelper()
+ .execute(
+ "GET",
+ new String[] {"requests", requestId, "commands", commandId},
+ new HashMap<>(),
+ null);
+ String content = StreamUtil.getStringFromStream(response.getContent());
+ String value = new JSONObject(content).getString("state");
+ return ClusterCommand.State.valueOf(value);
+ } catch (IOException | JSONException | IllegalArgumentException e) {
+ CLog.w("Failed to get state of request %s command %s", requestId, commandId);
+ CLog.e(e);
+ return ClusterCommand.State.UNKNOWN;
+ }
+ }
+
+ /**
+ * Parse command tasks from the http response
+ *
+ * @param httpResponse the http response
+ * @return a list of command tasks
+ * @throws IOException
+ */
+ private static List<ClusterCommand> parseCommandTasks(HttpResponse httpResponse)
+ throws IOException {
+ String response = "";
+ InputStream content = httpResponse.getContent();
+ if (content == null) {
+ throw new IOException("null response");
+ }
+ response = StreamUtil.getStringFromStream(content);
+ // Parse the response
+ try {
+ JSONObject jsonResponse = new JSONObject(response);
+ if (jsonResponse.has("tasks")) {
+ // Convert the JSON commands to ClusterCommand objects
+ JSONArray jsonCommands = jsonResponse.getJSONArray("tasks");
+ List<ClusterCommand> commandTasks = new ArrayList<>(jsonCommands.length());
+ for (int i = 0; i < jsonCommands.length(); i++) {
+ JSONObject jsonCommand = jsonCommands.getJSONObject(i);
+ commandTasks.add(ClusterCommand.fromJson(jsonCommand));
+ }
+ return commandTasks;
+ } else {
+ // No work to be done
+ return Collections.<ClusterCommand>emptyList();
+ }
+ } catch (JSONException e) {
+ // May be a server-side issue. Log a warning and we'll try again later.
+ CLog.w("Failed to parse response from server: %s", response);
+ return Collections.<ClusterCommand>emptyList();
+ }
+ }
+
+ /**
+ * Helper method to convert a list of {@link IClusterEvent}s into a json format that TFC can
+ * understand.
+ *
+ * @param events The list of events to convert
+ * @param key The key string to use to identify the list of events
+ */
+ private static JSONObject buildPostData(List<? extends IClusterEvent> events, String key)
+ throws JSONException {
+
+ JSONArray array = new JSONArray();
+ for (IClusterEvent event : events) {
+ array.put(event.toJSON());
+ }
+ JSONObject postData = new JSONObject();
+ postData.put(key, array);
+ return postData;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public IClusterEventUploader<ClusterCommandEvent> getCommandEventUploader() {
+ if (mCommandEventUploader == null) {
+ mCommandEventUploader = new ClusterCommandEventUploader();
+ }
+ return mCommandEventUploader;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public IClusterEventUploader<ClusterHostEvent> getHostEventUploader() {
+ if (mHostEventUploader == null) {
+ mHostEventUploader = new ClusterHostEventUploader();
+ }
+ return mHostEventUploader;
+ }
+
+ /**
+ * Get the shared {@link IRestApiHelper} instance.
+ *
+ * <p>Exposed for testing.
+ *
+ * @return the shared {@link IRestApiHelper} instance.
+ */
+ @VisibleForTesting
+ IRestApiHelper getApiHelper() {
+ if (mApiHelper == null) {
+ HttpTransport transport = null;
+ transport = new NetHttpTransport();
+ HttpRequestFactory requestFactory = transport.createRequestFactory();
+ String baseUrl = getClusterOptions().getServiceUrl();
+ if (baseUrl == null) {
+ baseUrl = DEFAULT_TFC_URL_BASE;
+ }
+ mApiHelper = new RestApiHelper(requestFactory, baseUrl);
+ }
+ return mApiHelper;
+ }
+
+ /** Get the {@link IClusterOptions} instance used to store cluster-related settings. */
+ IClusterOptions getClusterOptions() {
+ if (mClusterOptions == null) {
+ mClusterOptions =
+ (IClusterOptions)
+ GlobalConfiguration.getInstance()
+ .getConfigurationObject(ClusterOptions.TYPE_NAME);
+ if (mClusterOptions == null) {
+ throw new IllegalStateException(
+ "cluster_options not defined. You must add this "
+ + "object to your global config. See google/atp/cluster.xml.");
+ }
+ }
+ return mClusterOptions;
+ }
+
+ IHostOptions getHostOptions() {
+ if (mHostOptions == null) {
+ mHostOptions = GlobalConfiguration.getInstance().getHostOptions();
+ }
+ return mHostOptions;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterCommand.java b/src/com/android/tradefed/cluster/ClusterCommand.java
new file mode 100644
index 0000000..a821f99
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterCommand.java
@@ -0,0 +1,225 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.google.common.base.Strings;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/** A class that represents a task fetched from TF Cluster. */
+public class ClusterCommand {
+
+ public static enum RequestType {
+ /** An unmanaged request: the command line will run as is by the current TF process. */
+ UNMANAGED,
+ /** A managed request: the command line will run by a new TF process. */
+ MANAGED;
+ }
+
+ /** Command's status in the TF cluster. */
+ public static enum State {
+ /** Initial state, or failed to determine state. */
+ UNKNOWN,
+ /** Inserted into the cluster's queue. */
+ QUEUED,
+ /** Currently being executed. */
+ RUNNING,
+ /** Canceled by user, or failed to allocate a device. */
+ CANCELED,
+ /** Completed successfully. */
+ COMPLETED,
+ /** Completed exceptionally. */
+ ERROR,
+ /** Non-retryable error, e.g. invalid configuration. */
+ FATAL
+ }
+
+ private final String mTaskId;
+ private final String mRequestId;
+ private final String mCommandId;
+ private final String mCommandLine;
+ private final RequestType mRequestType;
+ private final Integer mShardCount;
+ private final Integer mShardIndex;
+ private final String mAttemptId;
+ // Devices try to match the current command.
+ private List<String> mTargetDeviceSerials = new ArrayList<>();
+
+ public ClusterCommand(String commandId, String taskId, String cmdLine) {
+ this(null, commandId, taskId, cmdLine, null, RequestType.UNMANAGED, null, null);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param requestId A request ID
+ * @param commandId The ID of the command that issued this task
+ * @param taskId The ID of this task
+ * @param cmdLine The command line to run
+ * @param requestType A request type
+ * @param shardCount A shard count
+ * @param shardIndex A shard index
+ */
+ public ClusterCommand(
+ String requestId,
+ String commandId,
+ String taskId,
+ String cmdLine,
+ String attemptId,
+ RequestType requestType,
+ Integer shardCount,
+ Integer shardIndex) {
+ mTaskId = taskId;
+ mRequestId = requestId;
+ mCommandId = commandId;
+ mCommandLine = cmdLine;
+ mRequestType = requestType;
+ mShardCount = shardCount;
+ mShardIndex = shardIndex;
+ if (!Strings.isNullOrEmpty(attemptId)) {
+ mAttemptId = attemptId;
+ } else {
+ // TODO(b/123294120): Remove creating attemptId on TF side once
+ // b/123294120 rolls out.
+ mAttemptId = UUID.randomUUID().toString();
+ }
+ }
+
+ /**
+ * Returns the task ID.
+ *
+ * @return task ID.
+ */
+ public String getTaskId() {
+ return mTaskId;
+ }
+
+ /**
+ * Returns the request ID.
+ *
+ * @return the request ID
+ */
+ public String getRequestId() {
+ return mRequestId;
+ }
+
+ /**
+ * Returns the command ID.
+ *
+ * @return the command ID
+ */
+ public String getCommandId() {
+ return mCommandId;
+ }
+
+ /**
+ * Returns the attempt ID. The attempt is randomly generated GUID used to distinguish multiple
+ * command runs.
+ *
+ * @return the attempt ID
+ */
+ public String getAttemptId() {
+ return mAttemptId;
+ }
+
+ /**
+ * Returns the command line string.
+ *
+ * @return the command line string.
+ */
+ public String getCommandLine() {
+ return mCommandLine;
+ }
+
+ /**
+ * Returns a request type
+ *
+ * @return a request type
+ */
+ public RequestType getRequestType() {
+ return mRequestType;
+ }
+
+ /**
+ * Returns the list of target device serials on which this command will attempt to run.
+ *
+ * @return the list of target device serials
+ */
+ public List<String> getTargetDeviceSerials() {
+ return mTargetDeviceSerials;
+ }
+
+ /**
+ * Sets the list of target device serials on which the command will try to run.
+ *
+ * @param targetDeviceSerials the list of device serials to set
+ */
+ public void setTargetDeviceSerials(List<String> targetDeviceSerials) {
+ this.mTargetDeviceSerials = targetDeviceSerials;
+ }
+
+ /**
+ * Returns a shard count.
+ *
+ * @return a shard count.
+ */
+ public Integer getShardCount() {
+ return mShardCount;
+ }
+
+ /**
+ * Returns a shard index.
+ *
+ * @return a shard index.
+ */
+ public Integer getShardIndex() {
+ return mShardIndex;
+ }
+
+ private static Integer optInteger(JSONObject json, String name) throws JSONException {
+ if (json.isNull(name)) {
+ return null;
+ }
+ return json.getInt(name);
+ }
+
+ public static ClusterCommand fromJson(JSONObject json) throws JSONException {
+ ClusterCommand command =
+ new ClusterCommand(
+ json.getString("request_id"),
+ json.getString("command_id"),
+ json.getString("task_id"),
+ json.getString("command_line"),
+ json.optString("attempt_id", null),
+ RequestType.valueOf(
+ json.optString("request_type", RequestType.UNMANAGED.name())),
+ optInteger(json, "shard_count"),
+ optInteger(json, "shard_index"));
+ JSONArray jsonDeviceSerials = json.optJSONArray("device_serials");
+ if (jsonDeviceSerials != null) {
+ final List<String> deviceSerials = new ArrayList<>();
+ for (int j = 0; j < jsonDeviceSerials.length(); j++) {
+ deviceSerials.add(jsonDeviceSerials.getString(j));
+ }
+ command.setTargetDeviceSerials(deviceSerials);
+ }
+ return command;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterCommandConfigBuilder.java b/src/com/android/tradefed/cluster/ClusterCommandConfigBuilder.java
new file mode 100644
index 0000000..8669f0b
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterCommandConfigBuilder.java
@@ -0,0 +1,278 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.config.ArgsOptionParser;
+import com.android.tradefed.config.Configuration;
+import com.android.tradefed.config.ConfigurationException;
+import com.android.tradefed.config.DeviceConfigurationHolder;
+import com.android.tradefed.config.IConfiguration;
+import com.android.tradefed.config.IDeviceConfiguration;
+import com.android.tradefed.log.SimpleFileLogger;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.util.MultiMap;
+import com.android.tradefed.util.StringUtil;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/** A class to build a configuration file for a cluster command. */
+public class ClusterCommandConfigBuilder {
+
+ private static final String TEST_TAG = "cluster_command_launcher";
+
+ private ClusterCommand mCommand;
+ private TestEnvironment mTestEnvironment;
+ private List<TestResource> mTestResources;
+ private TestContext mTestContext;
+ private File mWorkDir;
+
+ /**
+ * Set a {@link ClusterCommand} object.
+ *
+ * @param command a {@link ClusterCommand} object.
+ * @return {@link ClusterCommandConfigBuilder} for chaining.
+ */
+ public ClusterCommandConfigBuilder setClusterCommand(ClusterCommand command) {
+ mCommand = command;
+ return this;
+ }
+
+ /**
+ * Set a {@link TestEnvironment} object.
+ *
+ * @param testEnvironment a {@link TestEnvironment} object.
+ * @return {@link ClusterCommandConfigBuilder} for chaining.
+ */
+ public ClusterCommandConfigBuilder setTestEnvironment(TestEnvironment testEnvironment) {
+ mTestEnvironment = testEnvironment;
+ return this;
+ }
+
+ /**
+ * Set a list of {@link TestResource} object.
+ *
+ * @param testResources a list of {@link TestResource} objects.
+ * @return {@link ClusterCommandConfigBuilder} for chaining.
+ */
+ public ClusterCommandConfigBuilder setTestResources(List<TestResource> testResources) {
+ mTestResources = testResources;
+ return this;
+ }
+
+ /**
+ * Set a {@link TestContext} object.
+ *
+ * @param testContext a {@link TestContext} object.
+ * @return {@link ClusterCommandConfigBuilder} for chaining.
+ */
+ public ClusterCommandConfigBuilder setTestContext(TestContext testContext) {
+ mTestContext = testContext;
+ return this;
+ }
+
+ /**
+ * Set a work directory for a command.
+ *
+ * @param workDir a work directory.
+ * @return {@link ClusterCommandConfigBuilder} for chaining.
+ */
+ public ClusterCommandConfigBuilder setWorkDir(File workDir) {
+ mWorkDir = workDir;
+ return this;
+ }
+
+ /** Get a {@link IConfiguration} object type name for {@link TradefedConfigObject.Type}. */
+ private String getConfigObjectTypeName(TradefedConfigObject.Type type) {
+ switch (type) {
+ case TARGET_PREPARER:
+ return Configuration.TARGET_PREPARER_TYPE_NAME;
+ case RESULT_REPORTER:
+ return Configuration.RESULT_REPORTER_TYPE_NAME;
+ default:
+ throw new UnsupportedOperationException(String.format("%s is not supported", type));
+ }
+ }
+
+ /** Create a {@link IConfiguration} object for a {@link TradefedConfigObject}. */
+ private Object createConfigObject(
+ TradefedConfigObject configObjDef, Map<String, String> envVars)
+ throws ConfigurationException {
+ Object configObj = null;
+ try {
+ configObj = Class.forName(configObjDef.getClassName()).newInstance();
+ } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+ throw new ConfigurationException(
+ String.format(
+ "Failed to add a config object '%s'", configObjDef.getClassName()),
+ e);
+ }
+ MultiMap<String, String> optionValues = configObjDef.getOptionValues();
+ List<String> optionArgs = new ArrayList<>();
+ for (String name : optionValues.keySet()) {
+ List<String> values = optionValues.get(name);
+ for (String value : values) {
+ optionArgs.add(String.format("--%s", name));
+ if (value != null) {
+ // value can be null for valueless options.
+ optionArgs.add(StringUtil.expand(value, envVars));
+ }
+ }
+ }
+ ArgsOptionParser parser = new ArgsOptionParser(configObj);
+ parser.parse(optionArgs);
+ return configObj;
+ }
+
+ /**
+ * Builds a configuration file.
+ *
+ * @return a {@link File} object for a generated configuration file.
+ */
+ public File build() throws ConfigurationException, IOException {
+ assert mCommand != null;
+ assert mTestEnvironment != null;
+ assert mTestResources != null;
+ assert mWorkDir != null;
+
+ IConfiguration config = new Configuration("Cluster Command " + mCommand.getCommandId(), "");
+ config.getCommandOptions().setTestTag(TEST_TAG);
+ List<IDeviceConfiguration> deviceConfigs = new ArrayList<>();
+ int index = 0;
+ assert 0 < mCommand.getTargetDeviceSerials().size();
+
+ // Split config defs into device/non-device ones.
+ List<TradefedConfigObject> deviceConfigObjDefs = new ArrayList<>();
+ List<TradefedConfigObject> nonDeviceConfigObjDefs = new ArrayList<>();
+ for (TradefedConfigObject configObjDef : mTestEnvironment.getTradefedConfigObjects()) {
+ if (TradefedConfigObject.Type.TARGET_PREPARER.equals(configObjDef.getType())) {
+ deviceConfigObjDefs.add(configObjDef);
+ } else {
+ nonDeviceConfigObjDefs.add(configObjDef);
+ }
+ }
+
+ Map<String, String> envVars = new TreeMap<>();
+ envVars.put("TF_WORK_DIR", mWorkDir.getAbsolutePath());
+ envVars.putAll(mTestEnvironment.getEnvVars());
+ envVars.putAll(mTestContext.getEnvVars());
+ for (String serial : mCommand.getTargetDeviceSerials()) {
+ IDeviceConfiguration device =
+ new DeviceConfigurationHolder(String.format("TF_DEVICE_%d", index++));
+ device.getDeviceRequirements().setSerial(serial);
+ for (TradefedConfigObject configObjDef : deviceConfigObjDefs) {
+ device.addSpecificConfig(createConfigObject(configObjDef, envVars));
+ }
+ deviceConfigs.add(device);
+ }
+ deviceConfigs.get(0).addSpecificConfig(new ClusterBuildProvider());
+ config.setDeviceConfigList(deviceConfigs);
+ config.setTest(new ClusterCommandLauncher());
+ config.setLogSaver(new ClusterLogSaver());
+ // TODO(b/135636270): return log path to TFC instead of relying on a specific filename
+ config.setLogOutput(new SimpleFileLogger());
+ config.injectOptionValue(
+ "simple-file:path",
+ Paths.get(mWorkDir.getAbsolutePath(), "logs", "host_log.txt").toString());
+ config.setTestInvocationListeners(Collections.<ITestInvocationListener>emptyList());
+ for (TradefedConfigObject configObjDef : nonDeviceConfigObjDefs) {
+ String typeName = getConfigObjectTypeName(configObjDef.getType());
+ @SuppressWarnings("unchecked")
+ List<Object> configObjs = (List<Object>) config.getConfigurationObjectList(typeName);
+ configObjs.add(createConfigObject(configObjDef, envVars));
+ config.setConfigurationObjectList(typeName, configObjs);
+ }
+
+ config.injectOptionValue("cluster:request-id", mCommand.getRequestId());
+ config.injectOptionValue("cluster:command-id", mCommand.getCommandId());
+ config.injectOptionValue("cluster:attempt-id", mCommand.getAttemptId());
+ // FIXME: Make this configurable.
+ config.injectOptionValue("enable-root", "false");
+
+ String commandLine = mTestContext.getCommandLine();
+ if (commandLine == null || commandLine.isEmpty()) {
+ commandLine = mCommand.getCommandLine();
+ }
+ config.injectOptionValue("cluster:command-line", commandLine);
+ config.injectOptionValue("cluster:original-command-line", mCommand.getCommandLine());
+ config.injectOptionValue("cluster:root-dir", mWorkDir.getAbsolutePath());
+
+ for (final Map.Entry<String, String> entry : envVars.entrySet()) {
+ config.injectOptionValue("cluster:env-var", entry.getKey(), entry.getValue());
+ }
+ for (final String script : mTestEnvironment.getSetupScripts()) {
+ config.injectOptionValue("cluster:setup-script", script);
+ }
+ if (mTestEnvironment.useSubprocessReporting()) {
+ config.injectOptionValue("cluster:use-subprocess-reporting", "true");
+ }
+ config.injectOptionValue(
+ "cluster:output-idle-timeout",
+ String.valueOf(mTestEnvironment.getOutputIdleTimeout()));
+ for (final Map.Entry<String, String> entry :
+ mTestEnvironment.getJavaProperties().entrySet()) {
+ config.injectOptionValue("cluster:java-property", entry.getKey(), entry.getValue());
+ }
+ if (mTestEnvironment.getOutputFileUploadUrl() != null) {
+ String baseUrl = mTestEnvironment.getOutputFileUploadUrl();
+ if (!baseUrl.endsWith("/")) {
+ baseUrl += "/";
+ }
+ final String url =
+ String.format(
+ "%s%s/%s/", baseUrl, mCommand.getCommandId(), mCommand.getAttemptId());
+ config.injectOptionValue("cluster:output-file-upload-url", url);
+ }
+ for (final String pattern : mTestEnvironment.getOutputFilePatterns()) {
+ config.injectOptionValue("cluster:output-file-pattern", pattern);
+ }
+ if (mTestEnvironment.getContextFilePattern() != null) {
+ config.injectOptionValue(
+ "cluster:context-file-pattern", mTestEnvironment.getContextFilePattern());
+ }
+ for (String file : mTestEnvironment.getExtraContextFiles()) {
+ config.injectOptionValue("cluster:extra-context-file", file);
+ }
+ if (mTestEnvironment.getRetryCommandLine() != null) {
+ config.injectOptionValue(
+ "cluster:retry-command-line", mTestEnvironment.getRetryCommandLine());
+ }
+ if (mTestEnvironment.getLogLevel() != null) {
+ config.injectOptionValue("log-level", mTestEnvironment.getLogLevel());
+ }
+
+ List<TestResource> testResources = new ArrayList<>();
+ testResources.addAll(mTestResources);
+ testResources.addAll(mTestContext.getTestResources());
+ for (final TestResource resource : testResources) {
+ config.injectOptionValue(
+ "cluster:test-resource", resource.getName(), resource.getUrl());
+ }
+
+ File f = new File(mWorkDir, "command.xml");
+ PrintWriter writer = new PrintWriter(f);
+ config.dumpXml(writer);
+ writer.close();
+ return f;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterCommandEvent.java b/src/com/android/tradefed/cluster/ClusterCommandEvent.java
new file mode 100644
index 0000000..2e90da2
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterCommandEvent.java
@@ -0,0 +1,236 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.log.LogUtil.CLog;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/** A class to encapsulate cluster command events to be uploaded. */
+public class ClusterCommandEvent implements IClusterEvent {
+
+ public static final String DATA_KEY_ERROR = "error";
+ public static final String DATA_KEY_SUMMARY = "summary";
+ public static final String DATA_KEY_SETUP_TIME_MILLIS = "setup_time_millis";
+ public static final String DATA_KEY_FETCH_BUILD_TIME_MILLIS = "fetch_build_time_millis";
+ public static final String DATA_KEY_TOTAL_TEST_COUNT = "total_test_count";
+ public static final String DATA_KEY_FAILED_TEST_COUNT = "failed_test_count";
+ public static final String DATA_KEY_FAILED_TEST_RUN_COUNT = "failed_test_run_count";
+ public static final String EVENT_QUEUE = "command-event-queue";
+
+ // Maximum size of an individual data string value.
+ public static final int MAX_DATA_STRING_SIZE = 4095;
+
+ public enum Type {
+ AllocationFailed,
+ ConfigurationError,
+ FetchFailed,
+ ExecuteFailed,
+ InvocationInitiated,
+ InvocationStarted,
+ InvocationFailed,
+ InvocationEnded,
+ InvocationCompleted,
+ TestRunInProgress,
+ TestEnded
+ }
+
+ private long mTimestamp;
+ private Type mType;
+ private String mCommandTaskId;
+ private String mAttemptId;
+ private String mHostName;
+ private InvocationStatus mInvocationStatus;
+ private Map<String, Object> mData = new HashMap<>();
+ private Set<String> mDeviceSerials;
+
+ private ClusterCommandEvent() {}
+
+ public String getHostName() {
+ return mHostName;
+ }
+
+ public long getTimestamp() {
+ return mTimestamp;
+ }
+
+ public Type getType() {
+ return mType;
+ }
+
+ public String getCommandTaskId() {
+ return mCommandTaskId;
+ }
+
+ public String getAttemptId() {
+ return mAttemptId;
+ }
+
+ public InvocationStatus getInvocationStatus() {
+ return mInvocationStatus;
+ }
+
+ public Map<String, Object> getData() {
+ return mData;
+ }
+
+ public Set<String> getDeviceSerials() {
+ return mDeviceSerials;
+ }
+
+ public static class Builder {
+
+ private long mTimestamp = System.currentTimeMillis();
+ private Type mType;
+ private String mCommandTaskId;
+ private String mAttemptId;
+ private String mHostName;
+ private InvocationStatus mInvocationStatus;
+ private Map<String, Object> mData = new HashMap<>();
+ private Set<String> mDeviceSerials = new HashSet<>();
+
+ public Builder() {}
+
+ public Builder setTimestamp(final long timestamp) {
+ mTimestamp = timestamp;
+ return this;
+ }
+
+ public Builder setType(final Type type) {
+ mType = type;
+ return this;
+ }
+
+ public Builder setCommandTaskId(final String commandTaskId) {
+ mCommandTaskId = commandTaskId;
+ return this;
+ }
+
+ public Builder setAttemptId(final String attemptId) {
+ mAttemptId = attemptId;
+ return this;
+ }
+
+ public Builder setHostName(final String hostName) {
+ mHostName = hostName;
+ return this;
+ }
+
+ public Builder setInvocationStatus(final InvocationStatus invocationStatus) {
+ mInvocationStatus = invocationStatus;
+ return this;
+ }
+
+ public Builder setData(final String name, final Object value) {
+ if (value instanceof String && ((String) value).length() > MAX_DATA_STRING_SIZE) {
+ CLog.w(
+ String.format(
+ "Data for '%s' exceeds %d characters, and has been truncated.",
+ name, MAX_DATA_STRING_SIZE));
+ mData.put(name, ((String) value).substring(0, MAX_DATA_STRING_SIZE));
+ } else {
+ mData.put(name, value);
+ }
+ return this;
+ }
+
+ public Builder setDeviceSerials(final Set<String> deviceSerials) {
+ mDeviceSerials = deviceSerials;
+ return this;
+ }
+
+ public Builder addDeviceSerial(final String deviceSerial) {
+ mDeviceSerials.add(deviceSerial);
+ return this;
+ }
+
+ public ClusterCommandEvent build() {
+ final ClusterCommandEvent obj = new ClusterCommandEvent();
+ obj.mTimestamp = mTimestamp;
+ obj.mType = mType;
+ obj.mCommandTaskId = mCommandTaskId;
+ obj.mAttemptId = mAttemptId;
+ obj.mHostName = mHostName;
+ obj.mInvocationStatus = mInvocationStatus;
+ obj.mData = new HashMap<>(mData);
+ obj.mDeviceSerials = mDeviceSerials;
+ return obj;
+ }
+ }
+
+ /**
+ * Creates a base {@link Builder}.
+ *
+ * @return a {@link Builder}.
+ */
+ public static Builder createEventBuilder() {
+ return createEventBuilder(null);
+ }
+
+ /**
+ * Creates a base {@link Builder} for the given {@link ClusterCommand}.
+ *
+ * @return a {@link Builder}.
+ */
+ public static Builder createEventBuilder(final ClusterCommand command) {
+ final ClusterCommandEvent.Builder builder = new ClusterCommandEvent.Builder();
+ if (command != null) {
+ builder.setCommandTaskId(command.getTaskId());
+ builder.setAttemptId(command.getAttemptId());
+ }
+ return builder;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public JSONObject toJSON() throws JSONException {
+ final JSONObject json = new JSONObject();
+ json.put("type", this.getType().toString());
+ // event time should be in POSIX timestamp.
+ json.put("time", this.getTimestamp() / 1000);
+ json.put("task_id", this.getCommandTaskId());
+ json.put("attempt_id", this.getAttemptId());
+ json.put("hostname", this.getHostName());
+ // TODO(b/79583735): deprecated.
+ if (!this.getDeviceSerials().isEmpty()) {
+ json.put("device_serial", this.getDeviceSerials().iterator().next());
+ }
+ json.put("device_serials", new JSONArray(this.getDeviceSerials()));
+ if (mInvocationStatus != null) {
+ json.put("invocation_status", mInvocationStatus.toJSON());
+ }
+ json.put("data", new JSONObject(this.getData()));
+ return json;
+ }
+
+ @Override
+ public String toString() {
+ String str = null;
+ try {
+ str = toJSON().toString();
+ } catch (final JSONException e) {
+ // ignore
+ }
+ return str;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterCommandLauncher.java b/src/com/android/tradefed/cluster/ClusterCommandLauncher.java
new file mode 100644
index 0000000..a359a22
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterCommandLauncher.java
@@ -0,0 +1,363 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.annotations.VisibleForTesting;
+import com.android.helper.aoa.UsbDevice;
+import com.android.helper.aoa.UsbHelper;
+import com.android.tradefed.config.GlobalConfiguration;
+import com.android.tradefed.config.IConfiguration;
+import com.android.tradefed.config.IConfigurationReceiver;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.invoker.IInvocationContext;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.testtype.IInvocationContextReceiver;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.ArrayUtil;
+import com.android.tradefed.util.CommandResult;
+import com.android.tradefed.util.CommandStatus;
+import com.android.tradefed.util.FileIdleMonitor;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.IRunUtil;
+import com.android.tradefed.util.QuotationAwareTokenizer;
+import com.android.tradefed.util.RunUtil;
+import com.android.tradefed.util.StreamUtil;
+import com.android.tradefed.util.StringEscapeUtils;
+import com.android.tradefed.util.StringUtil;
+import com.android.tradefed.util.SubprocessTestResultsParser;
+import com.android.tradefed.util.SystemUtil;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+/**
+ * A {@link IRemoteTest} class to launch a command from TFC via a subprocess TF. FIXME: this needs
+ * to be extended to support multi-device tests.
+ */
+@OptionClass(alias = "cluster", global_namespace = false)
+public class ClusterCommandLauncher
+ implements IRemoteTest, IInvocationContextReceiver, IConfigurationReceiver {
+
+ public static final String TF_JAR_DIR = "TF_JAR_DIR";
+ public static final String TF_PATH = "TF_PATH";
+ public static final String TEST_WORK_DIR = "TEST_WORK_DIR";
+
+ private static final Duration MAX_EVENT_RECEIVER_WAIT_TIME = Duration.ofMinutes(10);
+
+ @Option(name = "root-dir", description = "A root directory", mandatory = true)
+ private File mRootDir;
+
+ @Option(name = "env-var", description = "Environment variables")
+ private Map<String, String> mEnvVars = new LinkedHashMap<>();
+
+ @Option(name = "setup-script", description = "Setup scripts")
+ private List<String> mSetupScripts = new ArrayList<>();
+
+ @Option(name = "script-timeout", description = "Script execution timeout", isTimeVal = true)
+ private long mScriptTimeout = 30 * 60 * 1000;
+
+ @Option(name = "java-property", description = "Java properties")
+ private Map<String, String> mJavaProperties = new LinkedHashMap<>();
+
+ @Option(name = "command-line", description = "A command line to launch.", mandatory = true)
+ private String mCommandLine = null;
+
+ @Option(
+ name = "original-command-line",
+ description =
+ "Original command line. It may differ from command-line in retry invocations.")
+ private String mOriginalCommandLine = null;
+
+ @Option(name = "use-subprocess-reporting", description = "Use subprocess reporting.")
+ private boolean mUseSubprocessReporting = false;
+
+ @Option(
+ name = "output-idle-timeout",
+ description = "Maximum time to wait for an idle subprocess",
+ isTimeVal = true)
+ private long mOutputIdleTimeout = 0L;
+
+ private IInvocationContext mInvocationContext;
+ private IConfiguration mConfiguration;
+ private IRunUtil mRunUtil;
+
+ @Override
+ public void setInvocationContext(IInvocationContext invocationContext) {
+ mInvocationContext = invocationContext;
+ }
+
+ @Override
+ public void setConfiguration(IConfiguration configuration) {
+ mConfiguration = configuration;
+ }
+
+ private String getEnvVar(String key) {
+ return getEnvVar(key, null);
+ }
+
+ private String getEnvVar(String key, String defaultValue) {
+ String value = mEnvVars.getOrDefault(key, defaultValue);
+ if (value != null) {
+ value = StringUtil.expand(value, mEnvVars);
+ }
+ return value;
+ }
+
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ // Pass jars under TF_PATH via classpath option (-cp)
+ String tfPath = getEnvVar(TF_PATH, System.getProperty(TF_JAR_DIR));
+ if (tfPath == null) {
+ throw new RuntimeException("cannot find TF path!");
+ }
+ final Set<String> jars = new LinkedHashSet<>();
+ for (final String path : tfPath.split(":")) {
+ jars.add(new File(path, "*").getAbsolutePath());
+ }
+
+ IRunUtil runUtil = getRunUtil();
+ runUtil.setWorkingDir(mRootDir);
+ // clear the TF_GLOBAL_CONFIG env, so another tradefed will not reuse the global config file
+ runUtil.unsetEnvVariable(GlobalConfiguration.GLOBAL_CONFIG_VARIABLE);
+ for (final String key : mEnvVars.keySet()) {
+ runUtil.setEnvVariable(key, getEnvVar(key));
+ }
+
+ final File testWorkDir = new File(getEnvVar(TEST_WORK_DIR, mRootDir.getAbsolutePath()));
+ final File logDir = new File(mRootDir, "logs");
+ logDir.mkdirs();
+ File stdoutFile = new File(logDir, "stdout.txt");
+ File stderrFile = new File(logDir, "stderr.txt");
+ FileIdleMonitor monitor = createFileMonitor(stdoutFile, stderrFile);
+
+ SubprocessTestResultsParser subprocessEventParser = null;
+ try (FileOutputStream stdout = new FileOutputStream(stdoutFile);
+ FileOutputStream stderr = new FileOutputStream(stderrFile)) {
+ long timeout = mScriptTimeout;
+ long startTime = System.currentTimeMillis();
+ for (String script : mSetupScripts) {
+ script = StringUtil.expand(script, mEnvVars);
+ CLog.i("Running a setup script: %s", script);
+ // FIXME: Refactor command execution into a helper function.
+ CommandResult result =
+ runUtil.runTimedCmd(
+ timeout,
+ stdout,
+ stderr,
+ QuotationAwareTokenizer.tokenizeLine(script));
+ if (!result.getStatus().equals(CommandStatus.SUCCESS)) {
+ String error = null;
+ if (result.getStatus().equals(CommandStatus.TIMED_OUT)) {
+ error = "timeout";
+ } else {
+ error = FileUtil.readStringFromFile(stderrFile);
+ }
+ throw new RuntimeException(String.format("Script failed to run: %s", error));
+ }
+ timeout -= (System.currentTimeMillis() - startTime);
+ if (timeout < 0) {
+ throw new RuntimeException(
+ String.format("Setup scripts failed to run in %sms", mScriptTimeout));
+ }
+ }
+
+ String classpath = ArrayUtil.join(":", jars);
+ String commandLine = mCommandLine;
+ if (classpath.isEmpty()) {
+ throw new RuntimeException(
+ String.format("cannot find any TF jars from %s!", tfPath));
+ }
+
+ if (mOriginalCommandLine != null && !mOriginalCommandLine.equals(commandLine)) {
+ // Make sure a wrapper XML of the original command is available because retries
+ // try to run original commands in Q+. If the original command was run with
+ // subprocess reporting, a recorded command would be one with .xml suffix.
+ new SubprocessConfigBuilder()
+ .setWorkingDir(testWorkDir)
+ .setOriginalConfig(
+ QuotationAwareTokenizer.tokenizeLine(mOriginalCommandLine)[0])
+ .build();
+ }
+ if (mUseSubprocessReporting) {
+ SubprocessReportingHelper mHelper = new SubprocessReportingHelper();
+ // Create standalone jar for subprocess result reporter, which is used
+ // for pre-O cts. The created jar is put in front position of the class path to
+ // override class with the same name.
+ classpath =
+ String.format(
+ "%s:%s",
+ mHelper.createSubprocessReporterJar(mRootDir).getAbsolutePath(),
+ classpath);
+ subprocessEventParser =
+ createSubprocessTestResultsParser(listener, true, mInvocationContext);
+ String port = Integer.toString(subprocessEventParser.getSocketServerPort());
+ commandLine = mHelper.buildNewCommandConfig(commandLine, port, testWorkDir);
+ }
+
+ List<String> javaCommandArgs = buildJavaCommandArgs(classpath, commandLine);
+ CLog.i("Running a command line: %s", commandLine);
+ CLog.i("args = %s", javaCommandArgs);
+ CLog.i("test working directory = %s", testWorkDir);
+
+ monitor.start();
+ runUtil.setWorkingDir(testWorkDir);
+ CommandResult result =
+ runUtil.runTimedCmd(
+ mConfiguration.getCommandOptions().getInvocationTimeout(),
+ stdout,
+ stderr,
+ javaCommandArgs.toArray(new String[javaCommandArgs.size()]));
+ if (!result.getStatus().equals(CommandStatus.SUCCESS)) {
+ String error = null;
+ if (result.getStatus().equals(CommandStatus.TIMED_OUT)) {
+ error = "timeout";
+ } else {
+ error = FileUtil.readStringFromFile(stderrFile);
+ }
+ throw new RuntimeException(String.format("Command failed to run: %s", error));
+ }
+ CLog.i("Successfully ran a command");
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ monitor.stop();
+ if (subprocessEventParser != null) {
+ subprocessEventParser.joinReceiver(
+ MAX_EVENT_RECEIVER_WAIT_TIME.toMillis(), /* wait for connection */ false);
+ StreamUtil.close(subprocessEventParser);
+ }
+ }
+ }
+
+ /** Build a shell command line to invoke a TF process. */
+ private List<String> buildJavaCommandArgs(String classpath, String tfCommandLine) {
+ // Build a command line to invoke a TF process.
+ final List<String> cmdArgs = new ArrayList<>();
+ cmdArgs.add(SystemUtil.getRunningJavaBinaryPath().getAbsolutePath());
+ cmdArgs.add("-cp");
+ cmdArgs.add(classpath);
+
+ // Pass Java properties as -D options.
+ for (final Entry<String, String> entry : mJavaProperties.entrySet()) {
+ cmdArgs.add(
+ String.format(
+ "-D%s=%s",
+ entry.getKey(), StringUtil.expand(entry.getValue(), mEnvVars)));
+ }
+ cmdArgs.add("com.android.tradefed.command.CommandRunner");
+ tfCommandLine = StringUtil.expand(tfCommandLine, mEnvVars);
+ cmdArgs.addAll(StringEscapeUtils.paramsToArgs(ArrayUtil.list(tfCommandLine)));
+
+ final Integer shardCount = mConfiguration.getCommandOptions().getShardCount();
+ final Integer shardIndex = mConfiguration.getCommandOptions().getShardIndex();
+ if (shardCount != null && shardCount > 1) {
+ cmdArgs.add("--shard-count");
+ cmdArgs.add(Integer.toString(shardCount));
+ if (shardIndex != null) {
+ cmdArgs.add("--shard-index");
+ cmdArgs.add(Integer.toString(shardIndex));
+ }
+ }
+
+ for (final ITestDevice device : mInvocationContext.getDevices()) {
+ // FIXME: Find a better way to support non-physical devices as well.
+ cmdArgs.add("--serial");
+ cmdArgs.add(device.getSerialNumber());
+ }
+
+ return cmdArgs;
+ }
+
+ /** Creates a file monitor which will perform a USB port reset if the subprocess is idle. */
+ private FileIdleMonitor createFileMonitor(File... files) {
+ // treat zero or negative timeout as infinite
+ long timeout = mOutputIdleTimeout > 0 ? mOutputIdleTimeout : Long.MAX_VALUE;
+ // reset USB ports if files are idle for too long
+ // TODO(peykov): consider making the callback customizable
+ return new FileIdleMonitor(Duration.ofMillis(timeout), this::resetUsbPorts, files);
+ }
+
+ /** Performs a USB port reset on all devices. */
+ private void resetUsbPorts() {
+ CLog.i("Subprocess output idle for %d ms, attempting USB port reset.", mOutputIdleTimeout);
+ try (UsbHelper usb = new UsbHelper()) {
+ for (String serial : mInvocationContext.getSerials()) {
+ try (UsbDevice device = usb.getDevice(serial)) {
+ if (device == null) {
+ CLog.w("Device '%s' not found during USB reset.", serial);
+ continue;
+ }
+ CLog.d("Resetting USB port for device '%s'", serial);
+ device.reset();
+ }
+ }
+ }
+ }
+
+ @VisibleForTesting
+ IRunUtil getRunUtil() {
+ if (mRunUtil == null) {
+ mRunUtil = new RunUtil();
+ }
+ return mRunUtil;
+ }
+
+ @VisibleForTesting
+ SubprocessTestResultsParser createSubprocessTestResultsParser(
+ ITestInvocationListener listener, boolean streaming, IInvocationContext context)
+ throws IOException {
+ return new SubprocessTestResultsParser(listener, streaming, context);
+ }
+
+ @VisibleForTesting
+ Map<String, String> getEnvVars() {
+ return mEnvVars;
+ }
+
+ @VisibleForTesting
+ List<String> getSetupScripts() {
+ return mSetupScripts;
+ }
+
+ @VisibleForTesting
+ Map<String, String> getJavaProperties() {
+ return mJavaProperties;
+ }
+
+ @VisibleForTesting
+ String getCommandLine() {
+ return mCommandLine;
+ }
+
+ @VisibleForTesting
+ boolean useSubprocessReporting() {
+ return mUseSubprocessReporting;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterCommandScheduler.java b/src/com/android/tradefed/cluster/ClusterCommandScheduler.java
new file mode 100644
index 0000000..d21d2f9
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterCommandScheduler.java
@@ -0,0 +1,718 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.build.BuildInfo;
+import com.android.tradefed.command.CommandScheduler;
+import com.android.tradefed.command.ICommandScheduler;
+import com.android.tradefed.command.remote.DeviceDescriptor;
+import com.android.tradefed.config.ConfigurationException;
+import com.android.tradefed.config.IConfiguration;
+import com.android.tradefed.device.DeviceAllocationState;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.FreeDeviceState;
+import com.android.tradefed.device.IDeviceManager;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.device.NoDeviceException;
+import com.android.tradefed.device.battery.BatteryController;
+import com.android.tradefed.device.battery.IBatteryInfo;
+import com.android.tradefed.device.battery.IBatteryInfo.BatteryState;
+import com.android.tradefed.invoker.IInvocationContext;
+import com.android.tradefed.invoker.InvocationContext;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.CollectingTestListener;
+import com.android.tradefed.result.ITestSummaryListener;
+import com.android.tradefed.result.TestRunResult;
+import com.android.tradefed.result.TestSummary;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.MultiMap;
+import com.android.tradefed.util.QuotationAwareTokenizer;
+
+import com.android.tradefed.cluster.ClusterHostEvent.HostEventType;
+import com.google.common.primitives.Ints;
+
+import org.json.JSONException;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A {@link ICommandScheduler} to support TFC (Tradefed Cluster). This scheduler runs commands from
+ * TFC command-queue and uploads invocation events to TFC command-event-queue.
+ */
+public class ClusterCommandScheduler extends CommandScheduler {
+
+ /** The {@link ScheduledThreadPoolExecutor} used to manage heartbeats. */
+ private ScheduledThreadPoolExecutor mHeartbeatThreadPool = null;
+
+ /** The {@link IClusterOptions} instance used to store cluster-related settings. */
+ private IClusterOptions mClusterOptions;
+
+ /** The {@link IClusterClient} instance used to interact with the TFC backend. */
+ private IClusterClient mClusterClient;
+
+ /**
+ * A {@link ThreadFactory} which returns threads in a dedicated heartbeat group.
+ *
+ * <p>This class is used as a factory by {@code mHeartbeatThreadPool} in order to segregate
+ * heartbeat threads from other "stray" threads to avoid tripping loose thread detection in
+ * {@link CommandScheduler}.
+ */
+ private static class HeartbeatThreadFactory implements ThreadFactory {
+ private static final ThreadGroup HB_GROUP;
+
+ static {
+ // fetch root thread group as this class may be initialized by an invocation thread
+ ThreadGroup tg = Thread.currentThread().getThreadGroup();
+ while (tg.getParent() != null) {
+ tg = tg.getParent();
+ }
+ HB_GROUP = new ThreadGroup(tg, "ClusterCommandScheduler.heartbeat");
+ }
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread thread = new Thread(HB_GROUP, r);
+ // heartbeat should always get cancelled, but ensure it doesn't prevent JVM exit
+ thread.setDaemon(true);
+ return thread;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void start() {
+ UploadHostEventWithState(HostState.RUNNING);
+ super.start();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void shutdown() {
+ UploadHostEventWithState(HostState.QUITTING);
+ getHeartbeatThreadPool().shutdown();
+ super.shutdown();
+ }
+
+ @Override
+ public synchronized void shutdownHard() {
+ UploadHostEventWithState(HostState.KILLING);
+ getHeartbeatThreadPool().shutdown();
+ super.shutdownHard();
+ }
+
+ /**
+ * A {@link com.android.tradefed.command.ICommandScheduler.IScheduledInvocationListener} to
+ * upload events to TFC.
+ */
+ class InvocationEventHandler extends CollectingTestListener
+ implements IScheduledInvocationListener, ITestSummaryListener {
+
+ private ScheduledFuture<?> mHeartbeat;
+ private final ClusterCommand mCommandTask;
+ private Set<String> mDeviceSerials = new HashSet<>();
+ private String mSummary;
+ private String mError;
+ private File mWorkDir;
+ private InvocationStatus mInvocationStatus;
+
+ /**
+ * Creates a {@link InvocationEventHandler} to track the given {@link ClusterCommand}.
+ *
+ * @param commandTask the {@link ClusterCommand} to track.
+ */
+ public InvocationEventHandler(ClusterCommand commandTask) {
+ mCommandTask = commandTask;
+ }
+
+ /**
+ * Sets a work directory for an invocation.
+ *
+ * @param dir a work directory.
+ */
+ public void setWorkDir(File dir) {
+ mWorkDir = dir;
+ }
+
+ private ClusterCommandEvent.Builder createEventBuilder() {
+ final ClusterCommandEvent.Builder builder =
+ ClusterCommandEvent.createEventBuilder(mCommandTask)
+ .setHostName(ClusterHostUtil.getHostName());
+ if (!mDeviceSerials.isEmpty()) {
+ builder.setDeviceSerials(mDeviceSerials);
+ }
+ return builder;
+ }
+
+ private void updateInvocationStatus() {
+ if (!getClusterOptions().shouldUploadInvocationStatus()) {
+ return;
+ }
+ final InvocationStatus obj = new InvocationStatus();
+ final Collection<TestRunResult> testRunResults = this.getMergedTestRunResults();
+ for (final TestRunResult result : testRunResults) {
+ final TestGroupStatus testGroupStatus =
+ new TestGroupStatus(
+ result.getName(),
+ result.getNumTests(),
+ result.getNumCompleteTests(),
+ result.getNumAllFailedTests(),
+ result.isRunComplete(),
+ result.getElapsedTime(),
+ result.getRunFailureMessage());
+ obj.addTestGroupStatus(testGroupStatus);
+ }
+ mInvocationStatus = obj;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void invocationInitiated(IInvocationContext context) {
+ for (ITestDevice device : context.getDevices()) {
+ mDeviceSerials.add(device.getSerialNumber());
+ }
+ final ClusterCommandEvent event =
+ createEventBuilder()
+ .setType(ClusterCommandEvent.Type.InvocationInitiated)
+ .build();
+ getClusterClient().getCommandEventUploader().postEvent(event);
+ getClusterClient().getCommandEventUploader().flush();
+ mHeartbeat = startHeartbeat();
+ // Check that devices are in charging state before starting the invocation.
+ for (ITestDevice device : context.getDevices()) {
+ try {
+ BatteryState state = BatteryController.getDeviceChargingState(device);
+ if (BatteryState.NOT_CHARGING.equals(state)) {
+ IBatteryInfo info = BatteryController.getBatteryInfoForDevice(device);
+ if (info != null) {
+ info.enableCharging(device);
+ }
+ }
+ } catch (DeviceNotAvailableException e) {
+ CLog.e(e);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void invocationStarted(IInvocationContext context) {
+ super.invocationStarted(context);
+ final ClusterCommandEvent event =
+ createEventBuilder()
+ .setType(ClusterCommandEvent.Type.InvocationStarted)
+ .build();
+ getClusterClient().getCommandEventUploader().postEvent(event);
+ getClusterClient().getCommandEventUploader().flush();
+ }
+
+ @Override
+ public void testRunStarted(String name, int numTests) {
+ testRunStarted(name, numTests, 0);
+ }
+
+ @Override
+ public void testRunStarted(String name, int numTests, int attemptNumber) {
+ testRunStarted(name, numTests, attemptNumber, System.currentTimeMillis());
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void testRunStarted(String name, int numTests, int attemptNumber, long startTime) {
+ super.testRunStarted(name, numTests, attemptNumber, startTime);
+ updateInvocationStatus();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void invocationFailed(Throwable cause) {
+ super.invocationFailed(cause);
+
+ mError = cause.toString();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void invocationEnded(long elapsedTime) {
+ super.invocationEnded(elapsedTime);
+
+ ClusterCommandEvent event =
+ createEventBuilder()
+ .setType(ClusterCommandEvent.Type.InvocationEnded)
+ .setData(ClusterCommandEvent.DATA_KEY_ERROR, mError)
+ .build();
+ getClusterClient().getCommandEventUploader().postEvent(event);
+ getClusterClient().getCommandEventUploader().flush();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void invocationComplete(
+ IInvocationContext metadata, Map<ITestDevice, FreeDeviceState> devicesStates) {
+ if (mWorkDir != null) {
+ FileUtil.recursiveDelete(mWorkDir);
+ }
+
+ // TODO: handle multi-device where only one of the build could be missing.
+ if (getPrimaryBuildInfo() == null && mError == null) {
+ mError = "build not found";
+ }
+
+ long fetchBuildTimeMillis = -1;
+ long setupTimeMillis = -1;
+ if (metadata != null && metadata.getInvocationTimingMetrics() != null) {
+ for (Entry<IInvocationContext.TimingEvent, Long> entry :
+ metadata.getInvocationTimingMetrics().entrySet()) {
+ switch (entry.getKey()) {
+ case FETCH_BUILD:
+ fetchBuildTimeMillis = entry.getValue();
+ break;
+ case SETUP:
+ setupTimeMillis = entry.getValue();
+ break;
+ }
+ }
+ }
+
+ // Stop heartbeat thread before sending InvocationCompleted event.
+ if (mHeartbeat != null) {
+ mHeartbeat.cancel(true);
+ }
+ updateInvocationStatus();
+ final ClusterCommandEvent event =
+ createEventBuilder()
+ .setType(ClusterCommandEvent.Type.InvocationCompleted)
+ .setInvocationStatus(mInvocationStatus)
+ .setData(ClusterCommandEvent.DATA_KEY_ERROR, mError)
+ .setData(ClusterCommandEvent.DATA_KEY_SUMMARY, mSummary)
+ .setData(
+ ClusterCommandEvent.DATA_KEY_FETCH_BUILD_TIME_MILLIS,
+ Long.toString(fetchBuildTimeMillis))
+ .setData(
+ ClusterCommandEvent.DATA_KEY_SETUP_TIME_MILLIS,
+ Long.toString(setupTimeMillis))
+ .setData(
+ ClusterCommandEvent.DATA_KEY_TOTAL_TEST_COUNT,
+ Integer.toString(getNumTotalTests()))
+ .setData(
+ ClusterCommandEvent.DATA_KEY_FAILED_TEST_COUNT,
+ Integer.toString(getNumAllFailedTests()))
+ .setData(
+ ClusterCommandEvent.DATA_KEY_FAILED_TEST_RUN_COUNT,
+ Integer.toString(getNumAllFailedTestRuns()))
+ .build();
+ getClusterClient().getCommandEventUploader().postEvent(event);
+ getClusterClient().getCommandEventUploader().flush();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void putSummary(List<TestSummary> summaries) {
+ final StringBuilder sb = new StringBuilder();
+ for (final TestSummary summary : summaries) {
+ sb.append(summary.getSummary());
+ sb.append("\n");
+ }
+ mSummary = sb.toString();
+ }
+
+ private ScheduledFuture<?> startHeartbeat() {
+ return getHeartbeatThreadPool()
+ .scheduleAtFixedRate(
+ new HeartbeatSender(),
+ 0,
+ getClusterOptions().getInvocationHeartbeatInterval(),
+ TimeUnit.MILLISECONDS);
+ }
+
+ class HeartbeatSender implements Runnable {
+ @Override
+ public void run() {
+ try {
+ // check cluster command's status
+ if (getClusterOptions().checkCommandState()) {
+ ClusterCommand.State status =
+ getClusterClient()
+ .getCommandState(
+ mCommandTask.getRequestId(),
+ mCommandTask.getCommandId());
+ if (ClusterCommand.State.CANCELED.equals(status)) {
+ CLog.w(
+ "Cluster marked command %s %s as canceled, stopping invocation",
+ mCommandTask.getRequestId(), mCommandTask.getCommandId());
+ Optional.ofNullable(getInvocationContext())
+ .map(IInvocationContext::getInvocationId)
+ .map(Ints::tryParse)
+ .ifPresent(ClusterCommandScheduler.this::stopInvocation);
+ }
+ }
+
+ final ClusterCommandEvent event =
+ createEventBuilder()
+ .setType(ClusterCommandEvent.Type.TestRunInProgress)
+ .setInvocationStatus(mInvocationStatus)
+ .build();
+ getClusterClient().getCommandEventUploader().postEvent(event);
+ } catch (Exception e) {
+ CLog.e("Error sending heartbeat to TFC:");
+ CLog.e(e);
+ }
+ }
+ }
+ }
+
+ synchronized ScheduledThreadPoolExecutor getHeartbeatThreadPool() {
+ if (mHeartbeatThreadPool == null) {
+ mHeartbeatThreadPool = new ScheduledThreadPoolExecutor(1, new HeartbeatThreadFactory());
+ // instead of throwing some exception on shutdown we simply log it.
+ mHeartbeatThreadPool.setRejectedExecutionHandler(
+ new RejectedExecutionHandler() {
+ @Override
+ public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
+ CLog.w(
+ "Rejecting Task %s rejected from executor %s",
+ r.toString(), e.toString());
+ }
+ });
+ // continue existing heartbeats after shutdown (until invocation is complete)
+ mHeartbeatThreadPool.setContinueExistingPeriodicTasksAfterShutdownPolicy(true);
+ }
+ return mHeartbeatThreadPool;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ protected void processReadyCommands(IDeviceManager manager) {
+ super.processReadyCommands(manager);
+
+ if (isShuttingDown()) {
+ return;
+ }
+
+ List<ClusterCommand> commands = null;
+ MultiMap<String, DeviceDescriptor> devices = getAvailableDevices(manager);
+ if (devices.isEmpty()) {
+ CLog.d("No devices are available for testing.");
+ return;
+ }
+ // Lease command tasks through the leasehosttasks API.
+ // Here we get all devices (available or not), so TFC will analyze the device tree to
+ // decide which group is allocated and which group is available.
+ devices = getDevices(manager, false);
+ commands = fetchHostCommands(devices);
+ if (commands.isEmpty()) {
+ CLog.d("No commands available for testing.");
+ return;
+ }
+ execCommands(commands);
+ }
+
+ /**
+ * Returns a map containing available devices grouped by their types.
+ *
+ * @param manager a {@link IDeviceManager}.
+ * @return a {@link MultiMap} of String to DeviceDescriptor containing available devices.
+ */
+ MultiMap<String, DeviceDescriptor> getAvailableDevices(IDeviceManager manager) {
+ return getDevices(manager, true);
+ }
+
+ /**
+ * Returns a map containing devices grouped by their types.
+ *
+ * @param manager a {@link IDeviceManager}.
+ * @param availableOnly only return available devices or all devices.
+ * @return a {@link MultiMap} of String to DeviceDescriptor containing available devices.
+ */
+ MultiMap<String, DeviceDescriptor> getDevices(IDeviceManager manager, boolean availableOnly) {
+ // Getting available device types
+ final MultiMap<String, DeviceDescriptor> devices = new MultiMap<>();
+ for (final DeviceDescriptor device : manager.listAllDevices()) {
+ if (availableOnly && device.getState() != DeviceAllocationState.Available) {
+ continue;
+ }
+ if (ClusterHostUtil.isIpPort(device.getSerial())) {
+ // Note(b/28802876): Skipping IP:PORT serials from cluster scheduling because they
+ // behave differently from physical devices and are not fully supported by TF.
+ continue;
+ }
+ String runTargetFormat = getClusterOptions().getRunTargetFormat();
+ String runTarget =
+ ClusterHostUtil.getRunTarget(
+ device, runTargetFormat, getClusterOptions().getDeviceTag());
+ CLog.d("%s is available", runTarget);
+ devices.put(runTarget, device);
+ }
+ return devices;
+ }
+
+ /**
+ * Get available flashing permits.
+ *
+ * @return the number of available flashing permits.
+ */
+ private int getAvailableFlashingPermits() {
+ // By default there is no limit on available flashing permits.
+ int availableFlashingPermits = Integer.MAX_VALUE;
+ final IClusterOptions options = getClusterOptions();
+
+ boolean checkFlashingPermitsLease = options.checkFlashingPermitsOnLease();
+ if (checkFlashingPermitsLease) {
+ availableFlashingPermits = getDeviceManager().getAvailableFlashingPermits();
+ CLog.i("available flasher permits %d", availableFlashingPermits);
+ }
+ return availableFlashingPermits;
+ }
+
+ /**
+ * Fetches commands for devices from the Tradefed Cluster's leasehosttasks API.
+ *
+ * @param devices a {@link MultiMap} of String to DeviceDescriptor containing devices.
+ * @return a list of {@link ClusterCommand}s.
+ */
+ List<ClusterCommand> fetchHostCommands(final MultiMap<String, DeviceDescriptor> devices) {
+ CLog.d("fetching cluster host commands from leasehosttasks...");
+ int availableFlashingPermits = getAvailableFlashingPermits();
+
+ // Don't try to lease if there are no flasher permits available
+ if (availableFlashingPermits == 0) {
+ CLog.i("There is no available flashing permits. Not lease any additional commands.");
+ return Collections.<ClusterCommand>emptyList();
+ }
+
+ final IClusterOptions options = getClusterOptions();
+ final MultiMap<String, String> deviceGroups = options.getDeviceGroup();
+ final Map<String, String> deviceToGroup = new HashMap<>();
+ for (String group : deviceGroups.keySet()) {
+ for (String deviceSerial : deviceGroups.get(group)) {
+ deviceToGroup.put(deviceSerial, group);
+ }
+ }
+ List<ClusterDeviceInfo> deviceInfos = new LinkedList<>();
+ for (String runTarget : devices.keySet()) {
+ for (DeviceDescriptor d : devices.get(runTarget)) {
+ String groupName = deviceToGroup.getOrDefault(d.getSerial(), null);
+ ClusterDeviceInfo deviceInfo =
+ new ClusterDeviceInfo.Builder()
+ .setDeviceDescriptor(d)
+ .setRunTarget(runTarget)
+ .setGroupName(groupName)
+ .build();
+ deviceInfos.add(deviceInfo);
+ }
+ }
+ try {
+ int count = Math.min(deviceInfos.size(), availableFlashingPermits);
+ List<ClusterCommand> commands =
+ getClusterClient()
+ .leaseHostCommands(
+ options.getClusterId(),
+ ClusterHostUtil.getHostName(),
+ deviceInfos,
+ options.getNextClusterIds(),
+ count);
+ return commands;
+ } catch (JSONException e) {
+ CLog.e(e);
+ return Collections.<ClusterCommand>emptyList();
+ }
+ }
+
+ /**
+ * Executes commands fetched from the cluster command queue.
+ *
+ * @param commands a list of {@link ClusterCommand}s fetched from the cluster command queue.
+ */
+ void execCommands(final List<ClusterCommand> commands) {
+ for (final ClusterCommand commandTask : commands) {
+ try {
+ final InvocationEventHandler handler = new InvocationEventHandler(commandTask);
+ switch (commandTask.getRequestType()) {
+ case UNMANAGED:
+ execClusterCommand(commandTask, handler);
+ break;
+ case MANAGED:
+ execManagedClusterCommand(commandTask, handler);
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ } catch (NoDeviceException e) {
+ CLog.w(
+ "no device meets requirements for cluster command [%s]; returning...",
+ commandTask.getTaskId());
+ CLog.w(e);
+ IClusterEventUploader<ClusterCommandEvent> eventUploader =
+ getClusterClient().getCommandEventUploader();
+ eventUploader.postEvent(
+ ClusterCommandEvent.createEventBuilder(commandTask)
+ .setHostName(ClusterHostUtil.getHostName())
+ .setType(ClusterCommandEvent.Type.AllocationFailed)
+ .build());
+ eventUploader.flush();
+ } catch (ConfigurationException | IOException | JSONException e) {
+ CLog.w("failed to execute cluster command [%s]: %s", commandTask.getTaskId(), e);
+ CLog.w(e);
+ IClusterEventUploader<ClusterCommandEvent> eventUploader =
+ getClusterClient().getCommandEventUploader();
+ eventUploader.postEvent(
+ ClusterCommandEvent.createEventBuilder(commandTask)
+ .setHostName(ClusterHostUtil.getHostName())
+ .setType(ClusterCommandEvent.Type.ConfigurationError)
+ .setData(ClusterCommandEvent.DATA_KEY_ERROR, e.toString())
+ .build());
+ eventUploader.flush();
+ }
+ }
+ }
+
+ void execClusterCommand(ClusterCommand commandTask, InvocationEventHandler handler)
+ throws ConfigurationException, IllegalArgumentException, NoDeviceException {
+ String cmdLine = commandTask.getCommandLine();
+ String[] args = QuotationAwareTokenizer.tokenizeLine(cmdLine);
+ // If it is a dry run command skip execution.
+ if (dryRunCommand(handler, args)) {
+ return;
+ }
+ // Append device serials to command.
+ // By assigning all applicable serials, TF will try one by one until allocation
+ // succeeds (or fails for all). This mitigates the issue where a single bad
+ // device can starve tests.
+ if (commandTask.getTargetDeviceSerials() != null) {
+ for (String serial : commandTask.getTargetDeviceSerials()) {
+ cmdLine += " --serial ";
+ cmdLine += serial;
+ }
+ }
+ CLog.i("executing cluster command: [%s] %s", commandTask.getTaskId(), cmdLine);
+ execCommand(handler, QuotationAwareTokenizer.tokenizeLine(cmdLine));
+ }
+
+ void execManagedClusterCommand(ClusterCommand commandTask, InvocationEventHandler handler)
+ throws IOException, JSONException, ConfigurationException, NoDeviceException {
+ File workDir = null;
+ try {
+ workDir = new File(System.getProperty("java.io.tmpdir"), commandTask.getAttemptId());
+ workDir.mkdirs();
+ final String requestId = commandTask.getRequestId();
+ final String commandId = commandTask.getCommandId();
+ final IClusterClient client = getClusterClient();
+ final TestEnvironment testEnvironment = client.getTestEnvironment(requestId);
+ final List<TestResource> testResources = client.getTestResources(requestId);
+ final TestContext testContext = client.getTestContext(requestId, commandId);
+ testResources.addAll(testContext.getTestResources());
+ final File configFile =
+ new ClusterCommandConfigBuilder()
+ .setWorkDir(workDir)
+ .setClusterCommand(commandTask)
+ .setTestEnvironment(testEnvironment)
+ .setTestResources(testResources)
+ .setTestContext(testContext)
+ .build();
+ CLog.i("executing cluster command: [%s] %s", commandTask.getTaskId(), configFile);
+ CLog.d("configFile: %s", FileUtil.readStringFromFile(configFile));
+ // FIXME: Find a way to upload a config file after an invocation is completed for
+ // debugging.
+ handler.setWorkDir(workDir);
+ execCommand(handler, new String[] {configFile.getAbsolutePath()});
+ // Unset workDir to avoid being cleaned up
+ workDir = null;
+ } finally {
+ if (workDir != null) {
+ FileUtil.recursiveDelete(workDir);
+ }
+ }
+ }
+
+ /**
+ * Determines if a given command is a dry-run. If the command is a dry-run, validate it. If
+ * there are any configs issue, it will throw a ConfigurationException.
+ *
+ * @param handler {@link InvocationEventHandler} to report events for dry-run validation.
+ * @param args the command to validate.
+ * @return true if the command are a dry run, false otherwise.
+ * @throws ConfigurationException
+ */
+ protected boolean dryRunCommand(final InvocationEventHandler handler, String[] args)
+ throws ConfigurationException {
+ IConfiguration config =
+ getConfigFactory().createConfigurationFromArgs(args, null, getKeyStoreClient());
+ if (config.getCommandOptions().isDryRunMode()) {
+ IInvocationContext context = new InvocationContext();
+ context.addDeviceBuildInfo("stub", new BuildInfo());
+ handler.invocationStarted(context);
+ config.validateOptions();
+ handler.invocationEnded(0);
+ IInvocationContext nullMeta = null;
+ handler.invocationComplete(nullMeta, null);
+ return true;
+ }
+ return false;
+ }
+
+ /** Get the {@link IClusterOptions} instance used to store cluster-related settings. */
+ IClusterOptions getClusterOptions() {
+ if (mClusterOptions == null) {
+ mClusterOptions = ClusterHostUtil.getClusterOptions();
+ }
+ return mClusterOptions;
+ }
+
+ /** Get the {@link IClusterClient} instance used to interact with the TFC backend. */
+ IClusterClient getClusterClient() {
+ if (mClusterClient == null) {
+ mClusterClient = ClusterHostUtil.getClusterClient();
+ }
+ return mClusterClient;
+ }
+
+ /** Event triggered, to upload host states */
+ private void UploadHostEventWithState(HostState state) {
+ try {
+ IClusterEventUploader<ClusterHostEvent> Uploader =
+ getClusterClient().getHostEventUploader();
+ ClusterHostEvent.Builder builder =
+ new ClusterHostEvent.Builder()
+ .setClusterId(getClusterOptions().getClusterId())
+ .setHostEventType(HostEventType.HostStateChanged)
+ .setHostName(ClusterHostUtil.getHostName())
+ .setHostState(state);
+ CLog.d("event uploading with state %s", state.toString());
+ ClusterHostEvent event = builder.build();
+ Uploader.postEvent(event);
+ CLog.d("event %s uploaded with state %s", event.toString(), state.toString());
+ Uploader.flush();
+ } catch (RuntimeException e) {
+ CLog.e("failed to upload host state %s to TFC: %s", state.toString(), e);
+ }
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterDeviceInfo.java b/src/com/android/tradefed/cluster/ClusterDeviceInfo.java
new file mode 100644
index 0000000..23c39eb
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterDeviceInfo.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.command.remote.DeviceDescriptor;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/** A class to encapsulate cluster device info to be uploaded. */
+public class ClusterDeviceInfo {
+ private String mRunTarget;
+ private String mGroupName;
+ private DeviceDescriptor mDeviceDescriptor;
+
+ private ClusterDeviceInfo(
+ DeviceDescriptor deviceDescriptor, String runTarget, String groupName) {
+ mDeviceDescriptor = deviceDescriptor;
+ mRunTarget = runTarget;
+ mGroupName = groupName;
+ }
+
+ public String getRunTarget() {
+ return mRunTarget;
+ }
+
+ public String getGroupName() {
+ return mGroupName;
+ }
+
+ public DeviceDescriptor getDeviceDescriptor() {
+ return mDeviceDescriptor;
+ }
+
+ public static class Builder {
+ private DeviceDescriptor mDeviceDescriptor;
+ private String mRunTarget;
+ private String mGroupName;
+
+ public Builder() {}
+
+ public Builder setRunTarget(final String runTarget) {
+ mRunTarget = runTarget;
+ return this;
+ }
+
+ public Builder setGroupName(final String groupName) {
+ mGroupName = groupName;
+ return this;
+ }
+
+ public Builder setDeviceDescriptor(final DeviceDescriptor deviceDescriptor) {
+ mDeviceDescriptor = deviceDescriptor;
+ return this;
+ }
+
+ public ClusterDeviceInfo build() {
+ final ClusterDeviceInfo deviceInfo =
+ new ClusterDeviceInfo(mDeviceDescriptor, mRunTarget, mGroupName);
+ return deviceInfo;
+ }
+ }
+
+ /**
+ * Generates the JSON Object for this device info.
+ *
+ * @return JSONObject equivalent of this device info.
+ * @throws JSONException
+ */
+ public JSONObject toJSON() throws JSONException {
+ final JSONObject json = new JSONObject();
+ json.put("device_serial", mDeviceDescriptor.getSerial());
+ json.put("run_target", mRunTarget);
+ json.put("build_id", mDeviceDescriptor.getBuildId());
+ json.put("product", mDeviceDescriptor.getProduct());
+ json.put("product_variant", mDeviceDescriptor.getProductVariant());
+ json.put("sdk_version", mDeviceDescriptor.getSdkVersion());
+ json.put("battery_level", mDeviceDescriptor.getBatteryLevel());
+ json.put("mac_address", mDeviceDescriptor.getMacAddress());
+ json.put("sim_state", mDeviceDescriptor.getSimState());
+ json.put("sim_operator", mDeviceDescriptor.getSimOperator());
+ json.put("state", mDeviceDescriptor.getState());
+ json.put("group_name", mGroupName);
+ return json;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterDeviceMonitor.java b/src/com/android/tradefed/cluster/ClusterDeviceMonitor.java
new file mode 100644
index 0000000..fc2ed9c
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterDeviceMonitor.java
@@ -0,0 +1,239 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import com.android.tradefed.command.remote.DeviceDescriptor;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.device.DeviceAllocationState;
+import com.android.tradefed.device.IDeviceMonitor;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.util.CommandResult;
+import com.android.tradefed.util.CommandStatus;
+import com.android.tradefed.util.IRunUtil;
+import com.android.tradefed.util.QuotationAwareTokenizer;
+import com.android.tradefed.util.RunUtil;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An {@link IDeviceMonitor} implementation that reports results to the Tradefed Cluster service.
+ */
+@OptionClass(alias = "cluster-device-monitor")
+public class ClusterDeviceMonitor implements IDeviceMonitor {
+
+ @Option(
+ name = "host-info-cmd",
+ description =
+ "A label and command to run periodically, to "
+ + "collect and send host info to the backend. May be repeated. Commands containing "
+ + "spaces should be double-quoted.")
+ private Map<String, String> mHostInfoCmds = new HashMap<String, String>();
+
+ private Map<String, String[]> mTokenizedHostInfoCmds = null;
+
+ @Option(
+ name = "host-info-cmd-timeout",
+ description =
+ "How long to wait for each "
+ + "host-info-cmd to complete, in millis. If the command times out, a (null) value "
+ + "will be passed to the backend for that particular command.")
+ private long mHostInfoCmdTimeout = 5 * 1000;
+
+ /** Worker thread to dispatch a cluster host event that includes a snapshot of the devices */
+ class EventDispatcher extends Thread {
+
+ private boolean mIsCanceled = false;
+ private IClusterEventUploader<ClusterHostEvent> mEventUploader = null;
+ private IClusterOptions mClusterOptions = null;
+
+ public EventDispatcher() {
+ super("ClusterDeviceMonitor.EventDispatcher");
+ this.setDaemon(true);
+ }
+
+ @Override
+ public void run() {
+ try {
+ while (!mIsCanceled) {
+ dispatch();
+ getRunUtil()
+ .sleep(
+ ClusterHostUtil.getClusterOptions()
+ .getDeviceMonitorSnapshotInterval());
+ }
+ } catch (Exception e) {
+ CLog.e(e);
+ }
+ }
+
+ IClusterEventUploader<ClusterHostEvent> getEventUploader() {
+ if (mEventUploader == null) {
+ mEventUploader = ClusterHostUtil.getClusterClient().getHostEventUploader();
+ }
+ return mEventUploader;
+ }
+
+ IClusterOptions getClusterOptions() {
+ if (mClusterOptions == null) {
+ mClusterOptions = ClusterHostUtil.getClusterOptions();
+ }
+ return mClusterOptions;
+ }
+
+ void dispatch() {
+ CLog.d("Start device snapshot.");
+ final IClusterEventUploader<ClusterHostEvent> eventUploader = getEventUploader();
+ final List<DeviceDescriptor> devices = listDevices();
+ final ClusterHostEvent.Builder builder =
+ new ClusterHostEvent.Builder()
+ .setHostEventType(ClusterHostEvent.HostEventType.DeviceSnapshot)
+ .setHostName(ClusterHostUtil.getHostName())
+ .setTfVersion(ClusterHostUtil.getTfVersion())
+ .setData(getAdditionalHostInfo())
+ .setClusterId(getClusterOptions().getClusterId())
+ .setNextClusterIds(getClusterOptions().getNextClusterIds());
+ for (DeviceDescriptor device : devices) {
+ if (device.isTemporary()) {
+ // Do not report temporary devices, they will go away when the invocation
+ // finish.
+ continue;
+ }
+ final ClusterDeviceInfo.Builder deviceBuilder = new ClusterDeviceInfo.Builder();
+ String runTargetFormat = getClusterOptions().getRunTargetFormat();
+ deviceBuilder.setDeviceDescriptor(device);
+ deviceBuilder.setRunTarget(
+ ClusterHostUtil.getRunTarget(
+ device, runTargetFormat, getClusterOptions().getDeviceTag()));
+
+ builder.addDeviceInfo(deviceBuilder.build());
+ }
+ // We want to force an upload.
+ CLog.d("Dispatched devicesnapshot.");
+ eventUploader.postEvent(builder.build());
+ eventUploader.flush();
+ }
+
+ void cancel() {
+ mIsCanceled = true;
+ }
+
+ boolean isCanceled() {
+ return mIsCanceled;
+ }
+ }
+
+ private EventDispatcher mDispatcher;
+ private DeviceLister mDeviceLister;
+
+ /** {@inheritDoc} */
+ @Override
+ public void run() {
+ if (ClusterHostUtil.getClusterOptions().isDeviceMonitorDisabled()) {
+ return;
+ }
+ mDispatcher = getEventDispatcher();
+ mDispatcher.start();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void stop() {
+ if (mDispatcher != null && mDispatcher.isAlive()) {
+ mDispatcher.cancel();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void setDeviceLister(DeviceLister lister) {
+ if (lister == null) {
+ throw new NullPointerException();
+ }
+ mDeviceLister = lister;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void notifyDeviceStateChange(
+ String serial, DeviceAllocationState oldState, DeviceAllocationState newState) {
+ // Nothing happens. We only take snapshots. Maybe we can add state change in the future.
+ }
+
+ @VisibleForTesting
+ EventDispatcher getEventDispatcher() {
+ if (mDispatcher == null) {
+ mDispatcher = new EventDispatcher();
+ }
+ return mDispatcher;
+ }
+
+ @VisibleForTesting
+ List<DeviceDescriptor> listDevices() {
+ return mDeviceLister.listDevices();
+ }
+
+ @VisibleForTesting
+ IRunUtil getRunUtil() {
+ return RunUtil.getDefault();
+ }
+
+ /** A helper method to tokenize the host info commands. */
+ void tokenizeCommands() {
+ if (mTokenizedHostInfoCmds != null && !mTokenizedHostInfoCmds.isEmpty()) {
+ // Commands already tokenized and cached. No need to tokenize again.
+ return;
+ }
+
+ mTokenizedHostInfoCmds = new HashMap<String, String[]>(mHostInfoCmds.size());
+ // Tokenize the commands and cache the result
+ for (Map.Entry<String, String> entry : mHostInfoCmds.entrySet()) {
+ final String key = entry.getKey();
+ final String cmd = entry.getValue();
+
+ CLog.d("Tokenized key %s command: %s", key, cmd);
+ mTokenizedHostInfoCmds.put(key, QuotationAwareTokenizer.tokenizeLine(cmd));
+ }
+ }
+
+ /** Queries additional host info from host-info-cmd options in TF configs. */
+ Map<String, String> getAdditionalHostInfo() {
+ final Map<String, String> info = new HashMap<>();
+ this.tokenizeCommands();
+
+ for (Map.Entry<String, String[]> entry : mTokenizedHostInfoCmds.entrySet()) {
+ final String key = entry.getKey();
+ final String[] cmd = entry.getValue();
+ final String cmdString = mHostInfoCmds.get(key);
+
+ final CommandResult result = getRunUtil().runTimedCmdSilently(mHostInfoCmdTimeout, cmd);
+
+ CLog.d("Command %s result: %s", cmdString, result.getStatus().toString());
+
+ if (result.getStatus() == CommandStatus.SUCCESS) {
+ info.put(key, result.getStdout());
+ } else {
+ info.put(key, result.getStderr());
+ }
+ }
+
+ return info;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterEventUploader.java b/src/com/android/tradefed/cluster/ClusterEventUploader.java
new file mode 100644
index 0000000..d239a9c
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterEventUploader.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.log.LogUtil.CLog;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+/** ClusterEventUploader class, which uploads {@link IClusterEvent} to TFC. */
+public abstract class ClusterEventUploader<T extends IClusterEvent>
+ implements IClusterEventUploader<T> {
+
+ // Default maximum event batch size
+ private static final int DEFAULT_MAX_BATCH_SIZE = 200;
+
+ // Default event upload interval in ms.
+ private static final long DEFAULT_EVENT_UPLOAD_INTERVAL = 60 * 1000;
+
+ private int mMaxBatchSize = DEFAULT_MAX_BATCH_SIZE;
+ private long mEventUploadInterval = DEFAULT_EVENT_UPLOAD_INTERVAL;
+ private long mLastEventUploadTime = 0;
+ private Queue<T> mEventQueue = new ConcurrentLinkedQueue<T>();
+
+ /** {@inheritDoc} */
+ @Override
+ public void setMaxBatchSize(int batchSize) {
+ mMaxBatchSize = batchSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int getMaxBatchSize() {
+ return mMaxBatchSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void setEventUploadInterval(long interval) {
+ mEventUploadInterval = interval;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getEventUploadInterval() {
+ return mEventUploadInterval;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void postEvent(final T event) {
+ mEventQueue.add(event);
+ uploadEvents(false);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void flush() {
+ uploadEvents(true);
+ }
+
+ /**
+ * Upload events.
+ *
+ * @param uploadNow upload now or wait for uploading with other events.
+ */
+ private void uploadEvents(final boolean uploadNow) {
+ final long now = System.currentTimeMillis();
+ if (!uploadNow && now - mLastEventUploadTime < getEventUploadInterval()) {
+ return;
+ }
+ uploadEvents();
+ }
+
+ /** Synchronized actually upload events. Only one thread will upload the events. */
+ private synchronized void uploadEvents() {
+ // Tradefed Cluster is unable to process command events larger than 100 KiB (b/72104215).
+ // The ClusterCommandEvent Builder provides some protection against this by limiting data
+ // fields to 1 KiB, but the number of data fields is not bounded, so we're not entirely
+ // protected against this.
+
+ mLastEventUploadTime = System.currentTimeMillis();
+ List<T> events = new ArrayList<T>();
+ try {
+ // Upload batches of events until there are no more left
+ while (!mEventQueue.isEmpty()) {
+ // Limit the number of events to upload at once
+ int batchSize = getMaxBatchSize();
+ while (!mEventQueue.isEmpty() && events.size() < batchSize) {
+ events.add(mEventQueue.poll());
+ }
+ doUploadEvents(events);
+ events.clear();
+ }
+ } catch (IOException e) {
+ CLog.w("failed to upload events: %s", e);
+ CLog.w("events will be uploaded with the next event.");
+ mEventQueue.addAll(events);
+ }
+ }
+
+ protected abstract void doUploadEvents(List<T> events) throws IOException;
+}
diff --git a/src/com/android/tradefed/cluster/ClusterHostEvent.java b/src/com/android/tradefed/cluster/ClusterHostEvent.java
new file mode 100644
index 0000000..4aa8ac9
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterHostEvent.java
@@ -0,0 +1,215 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.command.CommandScheduler;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** A class to encapsulate cluster host events to be uploaded. */
+public class ClusterHostEvent implements IClusterEvent {
+ private long mTimestamp;
+ private String mHostName;
+ private String mTfVersion;
+ private String mClusterId;
+ private List<String> mNextClusterIds;
+ private List<ClusterDeviceInfo> mDeviceInfos = new ArrayList<>();
+ private Map<String, String> mData = new HashMap<>();
+ public static final String EVENT_QUEUE = "host-event-queue";
+
+ /** Enums of the different types of host events. */
+ public enum HostEventType {
+ DeviceSnapshot("DeviceSnapshot", "DEVICE_SNAPSHOT"),
+ HostStateChanged("HostStateChanged", "HOST_STATE_CHANGED");
+ private final String mName;
+ // The AndroidEngProdAPIName should map to
+ // https://cs.corp.google.com/piper///depot/google3/google/internal/android/engprod/proto/test/v1/androidtest.proto?rcl=221372995&l=80
+ private final String mAndroidEngProdAPIName;
+
+ private HostEventType(final String name, final String androidEngProdAPIName) {
+ mName = name;
+ mAndroidEngProdAPIName = androidEngProdAPIName;
+ }
+
+ @Override
+ public String toString() {
+ return mName;
+ }
+
+ public String getAndroidEngProdAPIName() {
+ return mAndroidEngProdAPIName;
+ }
+ }
+
+ private HostEventType mType;
+ private CommandScheduler.HostState mHostState = CommandScheduler.HostState.UNKNOWN;
+
+ private ClusterHostEvent() {}
+
+ public long getTimestamp() {
+ return mTimestamp;
+ }
+
+ public String getHostName() {
+ return mHostName;
+ }
+
+ public String getTfVersion() {
+ return mTfVersion;
+ }
+
+ public String getClusterId() {
+ return mClusterId;
+ }
+
+ public CommandScheduler.HostState getHostState() {
+ return mHostState;
+ }
+
+ public long getTfStartTime() {
+ return ClusterHostUtil.getTfStartTimeMillis();
+ }
+
+ public List<ClusterDeviceInfo> getDeviceInfos() {
+ return mDeviceInfos;
+ }
+
+ public Map<String, String> getData() {
+ return mData;
+ }
+
+ public HostEventType getType() {
+ return mType;
+ }
+
+ public List<String> getNextClusterIds() {
+ return mNextClusterIds;
+ }
+
+ public static class Builder {
+ private HostEventType mType;
+ private long mTimestamp = System.currentTimeMillis();
+ private String mHostName;
+ private String mTfVersion;
+ private String mClusterId;
+ private List<String> mNextClusterIds;
+ private List<ClusterDeviceInfo> mDeviceInfos = new ArrayList<ClusterDeviceInfo>();
+ private Map<String, String> mData = new HashMap<>();
+ private CommandScheduler.HostState mHostState = CommandScheduler.HostState.UNKNOWN;
+
+ public Builder() {}
+
+ public Builder setHostEventType(final HostEventType type) {
+ mType = type;
+ return this;
+ }
+
+ public Builder setTimestamp(final long timestamp) {
+ mTimestamp = timestamp;
+ return this;
+ }
+
+ public Builder setClusterId(final String clusterId) {
+ mClusterId = clusterId;
+ return this;
+ }
+
+ public Builder setHostName(final String hostname) {
+ mHostName = hostname;
+ return this;
+ }
+
+ public Builder setTfVersion(final String tfVersion) {
+ mTfVersion = tfVersion;
+ return this;
+ }
+
+ public Builder addDeviceInfo(final ClusterDeviceInfo deviceInfo) {
+ mDeviceInfos.add(deviceInfo);
+ return this;
+ }
+
+ public Builder addDeviceInfos(List<ClusterDeviceInfo> deviceInfos) {
+ mDeviceInfos.addAll(deviceInfos);
+ return this;
+ }
+
+ public Builder setData(final String name, final String value) {
+ mData.put(name, value);
+ return this;
+ }
+
+ public Builder setData(Map<String, String> data) {
+ mData.putAll(data);
+ return this;
+ }
+
+ public Builder setNextClusterIds(List<String> nexClusterIds) {
+ mNextClusterIds = nexClusterIds;
+ return this;
+ }
+
+ public Builder setHostState(CommandScheduler.HostState state) {
+ mHostState = state;
+ return this;
+ }
+
+ public ClusterHostEvent build() {
+ final ClusterHostEvent event = new ClusterHostEvent();
+ event.mType = mType;
+ event.mTimestamp = mTimestamp;
+ event.mHostName = mHostName;
+ event.mTfVersion = mTfVersion;
+ event.mClusterId = mClusterId;
+ event.mDeviceInfos = new ArrayList<>(mDeviceInfos);
+ event.mData = new HashMap<>(mData);
+ event.mNextClusterIds = mNextClusterIds;
+ event.mHostState = mHostState;
+ return event;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public JSONObject toJSON() throws JSONException {
+ final JSONObject json = new JSONObject();
+ // event time should be in POSIX timestamp.
+ json.put("time", this.getTimestamp() / 1000);
+ if (this.getType() != null) json.put("type", this.getType().toString());
+ json.put("hostname", this.getHostName());
+ json.put("tf_version", this.getTfVersion());
+ json.put("cluster", this.getClusterId());
+ JSONArray deviceInfos = new JSONArray();
+ for (ClusterDeviceInfo d : this.getDeviceInfos()) {
+ deviceInfos.put(d.toJSON());
+ }
+ json.put("device_infos", deviceInfos);
+ json.put("data", new JSONObject(this.getData()));
+ if (this.getNextClusterIds() != null) {
+ json.put("next_cluster_ids", new JSONArray(this.getNextClusterIds()));
+ }
+ json.put("state", this.getHostState().toString());
+ json.put("tf_start_time_seconds", this.getTfStartTime() / 1000);
+ return json;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterHostUtil.java b/src/com/android/tradefed/cluster/ClusterHostUtil.java
new file mode 100644
index 0000000..1755c44
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterHostUtil.java
@@ -0,0 +1,228 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.command.remote.DeviceDescriptor;
+import com.android.tradefed.config.GlobalConfiguration;
+import com.android.tradefed.device.DeviceManager;
+import com.android.tradefed.device.DeviceManager.FastbootDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.util.VersionParser;
+
+import com.google.common.base.Strings;
+import com.google.common.net.HostAndPort;
+import com.google.common.net.InetAddresses;
+import com.google.common.primitives.Longs;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.security.InvalidParameterException;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/** Static util functions for TF Cluster to get global config instances, host information, etc. */
+public class ClusterHostUtil {
+
+ private static String sHostName = null;
+
+ static final String DEFAULT_TF_VERSION = "(unknown)";
+
+ private static long sTfStartTime = getCurrentTimeMillis();
+
+ /**
+ * Gets the hostname.
+ *
+ * @return the hostname or null if we were unable to fetch it
+ */
+ public static String getHostName() {
+ if (sHostName == null) {
+ try {
+ sHostName = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ CLog.w("failed to get hostname: %s", e);
+ }
+ }
+ return sHostName;
+ }
+
+ /**
+ * Gets the TF version running on this host.
+ *
+ * @return this host's TF version.
+ */
+ public static String getTfVersion() {
+ final String version = VersionParser.fetchVersion();
+ return toValidTfVersion(version);
+ }
+
+ /**
+ * Validates a TF version and returns it if it is OK.
+ *
+ * @param version The string for a TF version provided by {@link VersionParser}
+ * @return the version if valid or a default if not.
+ */
+ protected static String toValidTfVersion(String version) {
+ if (Strings.isNullOrEmpty(version) || Longs.tryParse(version) == null) {
+ // Making sure the version is valid. It should be a build number
+ return DEFAULT_TF_VERSION;
+ }
+ return version;
+ }
+
+ /**
+ * Returns the run target for a given device descriptor.
+ *
+ * @param device {@link DeviceDescriptor} to get run target for.
+ * @return run target.
+ */
+ public static String getRunTarget(
+ DeviceDescriptor device, String runTargetFormat, Map<String, String> deviceTags) {
+ if (runTargetFormat != null) {
+ // Make sure the pattern is non-greedy.
+ Pattern p = Pattern.compile("\\{([^:\\}]+)(:.*)?\\}");
+ Matcher m = p.matcher(runTargetFormat);
+ StringBuffer sb = new StringBuffer();
+ while (m.find()) {
+ String pattern = m.group(1);
+ String key = null;
+ String txt = null;
+ switch (pattern) {
+ case "PRODUCT":
+ txt = device.getProduct();
+ break;
+ case "PRODUCT_OR_DEVICE_CLASS":
+ // TODO: Refactor the logic to handle more flexible combinations.
+ txt = device.getProduct();
+ if (device.isStubDevice()) {
+ String deviceClass = device.getDeviceClass();
+ // If it's a fastboot device we report it as the product
+ if (!FastbootDevice.class.getSimpleName().equals(deviceClass)) {
+ txt = deviceClass;
+ }
+ }
+ break;
+ case "PRODUCT_VARIANT":
+ txt = device.getProductVariant();
+ break;
+ case "API_LEVEL":
+ txt = device.getSdkVersion();
+ break;
+ case "DEVICE_CLASS":
+ txt = device.getDeviceClass();
+ break;
+ case "SERIAL":
+ txt = device.getSerial();
+ break;
+ case "TAG":
+ if (deviceTags == null || deviceTags.isEmpty()) {
+ // simply delete the placeholder if there's nothing to match
+ txt = "";
+ } else {
+ txt = deviceTags.get(device.getSerial());
+ if (txt == null) {
+ txt = ""; // simply delete it if a tag does not exist
+ }
+ }
+ break;
+ case "DEVICE_PROP":
+ key = m.group(2).substring(1);
+ txt = device.getProperty(key);
+ break;
+ default:
+ throw new InvalidParameterException(
+ String.format(
+ "Unsupported pattern '%s' found for run target '%s'",
+ pattern, runTargetFormat));
+ }
+ if (txt == null || DeviceManager.UNKNOWN_DISPLAY_STRING.equals(txt)) {
+ CLog.i(
+ "No value found for pattern %s while formatting run target %s.",
+ pattern, runTargetFormat);
+ return DeviceManager.UNKNOWN_DISPLAY_STRING;
+ }
+ m.appendReplacement(sb, Matcher.quoteReplacement(txt));
+ }
+ m.appendTail(sb);
+ return sb.toString();
+ }
+ // Default behavior.
+ // TODO: Remove this when we cluster default run target is changed.
+ String runTarget = device.getProduct();
+ if (!runTarget.equals(device.getProductVariant())) {
+ runTarget += ":" + device.getProductVariant();
+ }
+ return runTarget;
+ }
+
+ /**
+ * Checks if a given input is a valid IP:PORT string.
+ *
+ * @param input a string to check
+ * @return true if the given input is an IP:PORT string
+ */
+ public static boolean isIpPort(String input) {
+ try {
+ HostAndPort hostAndPort = HostAndPort.fromString(input);
+ return InetAddresses.isInetAddress(hostAndPort.getHost());
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Returns the current system time.
+ *
+ * @return time in millis.
+ */
+ public static long getCurrentTimeMillis() {
+ return System.currentTimeMillis();
+ }
+
+ public static long getTfStartTimeMillis() {
+ return sTfStartTime;
+ }
+
+ /** Get the {@link IClusterOptions} instance used to store cluster-related settings. */
+ public static IClusterOptions getClusterOptions() {
+ IClusterOptions clusterOptions =
+ (IClusterOptions)
+ GlobalConfiguration.getInstance()
+ .getConfigurationObject(ClusterOptions.TYPE_NAME);
+ if (clusterOptions == null) {
+ throw new IllegalStateException(
+ "cluster_options not defined. You must add this "
+ + "object to your global config. See google/atp/cluster.xml.");
+ }
+
+ return clusterOptions;
+ }
+
+ /** Get the {@link IClusterClient} instance used to interact with the TFC backend. */
+ public static IClusterClient getClusterClient() {
+ IClusterClient ClusterClient =
+ (IClusterClient)
+ GlobalConfiguration.getInstance()
+ .getConfigurationObject(IClusterClient.TYPE_NAME);
+ if (ClusterClient == null) {
+ throw new IllegalStateException(
+ "cluster_client not defined. You must add this "
+ + "object to your global config. See google/atp/cluster.xml.");
+ }
+
+ return ClusterClient;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterLogSaver.java b/src/com/android/tradefed/cluster/ClusterLogSaver.java
new file mode 100644
index 0000000..f48717f
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterLogSaver.java
@@ -0,0 +1,427 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.annotations.VisibleForTesting;
+import com.android.tradefed.config.GlobalConfiguration;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.invoker.IInvocationContext;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ILogSaver;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.result.LogFile;
+import com.android.tradefed.result.LogFileSaver;
+import com.android.tradefed.util.FileUtil;
+
+import org.json.JSONException;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/** A {@link ILogSaver} class to upload test outputs to TFC. */
+@OptionClass(alias = "cluster", global_namespace = false)
+public class ClusterLogSaver implements ILogSaver {
+
+ /** A name of a text file containing all test output file names. */
+ public static final String FILE_NAMES_FILE_NAME = "FILES";
+
+ /** A name of a subdirectory containing all files generated by host process. */
+ public static final String TOOL_LOG_PATH = "tool-logs";
+
+ /** File picking strategies. */
+ public static enum FilePickingStrategy {
+ PICK_LAST,
+ PICK_FIRST
+ }
+
+ @Option(name = "root-dir", description = "A root directory", mandatory = true)
+ private File mRootDir;
+
+ @Option(name = "request-id", description = "A request ID", mandatory = true)
+ private String mRequestId;
+
+ @Option(name = "command-id", description = "A command ID", mandatory = true)
+ private String mCommandId;
+
+ @Option(name = "attempt-id", description = "A command attempt ID", mandatory = true)
+ private String mAttemptId;
+
+ @Option(
+ name = "output-file-upload-url",
+ description = "URL to upload output files to",
+ mandatory = true)
+ private String mOutputFileUploadUrl;
+
+ @Option(name = "output-file-pattern", description = "Output file patterns")
+ private List<String> mOutputFilePatterns = new ArrayList<>();
+
+ @Option(
+ name = "context-file-pattern",
+ description =
+ "A regex pattern for test context file(s). A test context file is a"
+ + " file to be used in successive invocations to pass context information.")
+ private String mContextFilePattern = null;
+
+ @Option(name = "file-picking-strategy", description = "A picking strategy for file(s)")
+ private FilePickingStrategy mFilePickingStrategy = FilePickingStrategy.PICK_LAST;
+
+ @Option(
+ name = "extra-context-file",
+ description =
+ "Additional files to include in the context file. "
+ + "Context file must be a ZIP archive.")
+ private List<String> mExtraContextFiles = new ArrayList<>();
+
+ @Option(
+ name = "retry-command-line",
+ description =
+ "A command line to store in test context. This will replace the original command line in a retry invocation.")
+ private String mRetryCommandLine = null;
+
+ private File mLogDir;
+ private LogFileSaver mLogFileSaver = null;
+ private IClusterClient mClusterClient = null;
+
+ @Override
+ public void invocationStarted(IInvocationContext context) {
+ mLogDir = new File(mRootDir, "logs");
+ mLogFileSaver = new LogFileSaver(mLogDir);
+ }
+
+ private Path getRelativePath(Path p) {
+ return mRootDir.toPath().relativize(p);
+ }
+
+ /** Returns a Path stream for all files under a directory matching a given pattern. */
+ private Stream<Path> getPathStream(final File dir, final Pattern pattern) throws IOException {
+ return Files.find(
+ dir.toPath(),
+ Integer.MAX_VALUE,
+ (path, attr) ->
+ attr.isRegularFile()
+ && pattern.matcher(getRelativePath(path).toString()).matches(),
+ FileVisitOption.FOLLOW_LINKS);
+ }
+
+ private Set<File> findFilesRecursively(final File dir, final String regex) {
+ final Pattern pattern = Pattern.compile(regex);
+ try (final Stream<Path> stream = getPathStream(dir, pattern)) {
+ return stream.map(Path::toFile).collect(Collectors.toSet());
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to collect output files", e);
+ }
+ }
+
+ private Set<String> getGroupNames(final String regex) {
+ final Set<String> names = new TreeSet<>();
+ Matcher m = Pattern.compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>").matcher(regex);
+ while (m.find()) {
+ names.add(m.group(1));
+ }
+ return names;
+ }
+
+ /**
+ * Find a test context file and collect environment vars if exists.
+ *
+ * <p>If there are multiple matches, it will only collect the first or last one in
+ * lexicographical order according to a given file picking strategy. If a newly collected
+ * environment var already exists in a given map, it will be overridden.
+ *
+ * @param dir a root directory.
+ * @param regex a regex pattern for a context file path. A relative path is used for matching.
+ * @param strategy a file picking strategy.
+ * @param envVars a map for environment vars.
+ * @return a {@link File} object.
+ */
+ @VisibleForTesting
+ File findTestContextFile(
+ final File dir,
+ final String regex,
+ final FilePickingStrategy strategy,
+ final Map<String, String> envVars) {
+ final Pattern pattern = Pattern.compile(regex);
+ try (Stream<Path> stream = getPathStream(dir, pattern)) {
+ Optional<Path> op = null;
+ switch (strategy) {
+ case PICK_FIRST:
+ op = stream.sorted().findFirst();
+ break;
+ case PICK_LAST:
+ op = stream.sorted(Comparator.reverseOrder()).findFirst();
+ break;
+ }
+ if (op == null || !op.isPresent()) {
+ return null;
+ }
+ final Path p = op.get();
+ Set<String> groupNames = getGroupNames(regex);
+ CLog.d("Context var names: %s", groupNames);
+ Path relPath = dir.toPath().relativize(p);
+ Matcher matcher = pattern.matcher(relPath.toString());
+ // One needs to call matches() before calling group() method.
+ matcher.matches();
+ for (final String name : groupNames) {
+ final String value = matcher.group(name);
+ if (value == null) {
+ continue;
+ }
+ envVars.put(name, value);
+ }
+ return p.toFile();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to collect a context file", e);
+ }
+ }
+
+ /** Determine a file's new path after applying an optional prefix. */
+ private String getDestinationPath(String prefix, File file) {
+ String filename = file.getName();
+ return prefix == null ? filename : Paths.get(prefix, filename).toString();
+ }
+
+ /**
+ * Create a text file containing a list of file names.
+ *
+ * @param filenames filenames to write.
+ * @param destFile a {@link File} where to write names to.
+ * @throws IOException if writing fails
+ */
+ private void writeFilenamesToFile(Set<String> filenames, File destFile) throws IOException {
+ String content = filenames.stream().sorted().collect(Collectors.joining("\n"));
+ FileUtil.writeToFile(content, destFile);
+ }
+
+ /**
+ * Upload files to mOutputFileUploadUrl.
+ *
+ * @param fileMap a {@link Map} of file and destination path string pairs.
+ * @return a {@link Map} of file and URL pairs.
+ */
+ private Map<File, String> uploadFiles(Map<File, String> fileMap, FilePickingStrategy strategy) {
+ // construct a map of unique destination paths and files, to prevent duplicate uploads
+ Map<String, File> destinationMap =
+ fileMap.entrySet()
+ .stream()
+ .sorted(Comparator.comparing(Map.Entry::getKey)) // sort by filename
+ .collect(
+ Collectors.toMap(
+ e -> getDestinationPath(e.getValue(), e.getKey()),
+ Map.Entry::getKey,
+ // use strategy if two files have the same destination
+ (first, second) ->
+ strategy == FilePickingStrategy.PICK_FIRST
+ ? first
+ : second));
+ fileMap.keySet().retainAll(destinationMap.values());
+ CLog.i("Collected %d files to upload", fileMap.size());
+ fileMap.keySet().forEach(f -> CLog.i(f.getAbsolutePath()));
+
+ // Create a file names file.
+ File fileNamesFile = new File(mRootDir, FILE_NAMES_FILE_NAME);
+ try {
+ writeFilenamesToFile(destinationMap.keySet(), fileNamesFile);
+ } catch (IOException e) {
+ CLog.e("Failed to write %s", fileNamesFile.getAbsolutePath());
+ }
+
+ final TestOutputUploader uploader = getTestOutputUploader();
+ try {
+ uploader.setUploadUrl(mOutputFileUploadUrl);
+ } catch (MalformedURLException e) {
+ throw new RuntimeException("Failed to set upload URL", e);
+ }
+
+ fileMap.put(fileNamesFile, null);
+ final Map<File, String> fileUrls = new TreeMap<>();
+ int index = 1;
+ for (Map.Entry<File, String> entry : fileMap.entrySet()) {
+ File file = entry.getKey();
+ CLog.i("Uploading file %d of %d: %s", index, fileMap.size(), file.getAbsolutePath());
+ try {
+ fileUrls.put(file, uploader.uploadFile(file, entry.getValue()));
+ } catch (IOException | RuntimeException e) {
+ CLog.e("Failed to upload %s: %s", file, e);
+ }
+ index++;
+ }
+ return fileUrls;
+ }
+
+ /** If the context file is a zip file, will append the specified files to it. */
+ @VisibleForTesting
+ void appendFilesToContext(File contextFile, List<String> filesToAdd) {
+ if (filesToAdd.isEmpty()) {
+ return;
+ }
+
+ // create new ZIP file system which allows creating files
+ URI uri = URI.create("jar:" + contextFile.toURI());
+ Map<String, String> env = new HashMap<>();
+ env.put("create", "true");
+ try (FileSystem zip = FileSystems.newFileSystem(uri, env)) {
+ // copy files into the zip file, will not overwrite existing files
+ for (String filename : filesToAdd) {
+ Path path = Paths.get(filename);
+ if (!path.isAbsolute()) {
+ path = mRootDir.toPath().resolve(path);
+ }
+ if (!path.toFile().exists()) {
+ CLog.w("File %s not found", path);
+ continue;
+ }
+ Path zipPath = zip.getPath(path.getFileName().toString());
+ Files.copy(path, zipPath);
+ }
+ } catch (IOException | RuntimeException e) {
+ CLog.w("Failed to append files to context");
+ CLog.e(e);
+ }
+ }
+
+ @Override
+ public void invocationEnded(long elapsedTime) {
+ // Key is the file to be uploaded. Value is the destination path to upload url.
+ // For example, to upload a.txt to uploadUrl/path1/, the destination path is "path1";
+ // To upload a.txt to uploadUrl/, the destination path is null.
+ final Map<File, String> outputFiles = new HashMap<>();
+ File contextFile = null;
+ Map<String, String> envVars = new TreeMap<>();
+
+ // Get a list of log files to upload (skip host_log_*.txt to prevent duplicate upload)
+ findFilesRecursively(mLogDir, "^((?!host_log_\\d+).)*$")
+ .forEach(file -> outputFiles.put(file, TOOL_LOG_PATH));
+
+ // Collect output files to upload
+ if (0 < mOutputFilePatterns.size()) {
+ final String regex =
+ mOutputFilePatterns
+ .stream()
+ .map((s) -> "(" + s + ")")
+ .collect(Collectors.joining("|"));
+ CLog.i("Collecting output files matching regex: " + regex);
+ findFilesRecursively(mRootDir, regex).forEach(file -> outputFiles.put(file, null));
+ }
+
+ // Collect a context file if exists.
+ if (mContextFilePattern != null) {
+ CLog.i("Collecting context file matching regex: " + mContextFilePattern);
+ contextFile =
+ findTestContextFile(
+ mRootDir, mContextFilePattern, mFilePickingStrategy, envVars);
+ if (contextFile != null) {
+ CLog.i("Context file = %s", contextFile.getAbsolutePath());
+ outputFiles.put(contextFile, null);
+ appendFilesToContext(contextFile, mExtraContextFiles);
+ } else {
+ CLog.i("No context file found");
+ }
+ }
+
+ final Map<File, String> outputFileUrls = uploadFiles(outputFiles, mFilePickingStrategy);
+ if (contextFile != null && outputFileUrls.containsKey(contextFile)) {
+ final IClusterClient client = getClusterClient();
+ final TestContext testContext = new TestContext();
+ testContext.setCommandLine(mRetryCommandLine);
+ testContext.addEnvVars(envVars);
+ final String name = getRelativePath(contextFile.toPath()).toString();
+ testContext.addTestResource(new TestResource(name, outputFileUrls.get(contextFile)));
+ try {
+ CLog.i("Updating test context: %s", testContext.toString());
+ client.updateTestContext(mRequestId, mCommandId, testContext);
+ } catch (IOException | JSONException e) {
+ throw new RuntimeException("failed to update test context", e);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ TestOutputUploader getTestOutputUploader() {
+ return new TestOutputUploader();
+ }
+
+ /** Get the {@link IClusterClient} instance used to interact with the TFC backend. */
+ @VisibleForTesting
+ IClusterClient getClusterClient() {
+ if (mClusterClient == null) {
+ mClusterClient =
+ (IClusterClient)
+ GlobalConfiguration.getInstance()
+ .getConfigurationObject(IClusterClient.TYPE_NAME);
+ if (mClusterClient == null) {
+ throw new IllegalStateException("cluster_client not defined in TF global config.");
+ }
+ }
+ return mClusterClient;
+ }
+
+ @Override
+ public LogFile saveLogData(String dataName, LogDataType dataType, InputStream dataStream)
+ throws IOException {
+ File log = mLogFileSaver.saveLogData(dataName, dataType, dataStream);
+ return new LogFile(log.getAbsolutePath(), null, dataType);
+ }
+
+ @Override
+ public LogFile saveLogDataRaw(String dataName, LogDataType dataType, InputStream dataStream)
+ throws IOException {
+ File log = mLogFileSaver.saveLogDataRaw(dataName, dataType.getFileExt(), dataStream);
+ return new LogFile(log.getAbsolutePath(), null, dataType);
+ }
+
+ @Override
+ public LogFile getLogReportDir() {
+ return new LogFile(mLogDir.getAbsolutePath(), null, LogDataType.DIR);
+ }
+
+ @VisibleForTesting
+ String getAttemptId() {
+ return mAttemptId;
+ }
+
+ @VisibleForTesting
+ String getOutputFileUploadUrl() {
+ return mOutputFileUploadUrl;
+ }
+
+ @VisibleForTesting
+ List<String> getOutputFilePatterns() {
+ return mOutputFilePatterns;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/ClusterOptions.java b/src/com/android/tradefed/cluster/ClusterOptions.java
new file mode 100644
index 0000000..e6d6a45
--- /dev/null
+++ b/src/com/android/tradefed/cluster/ClusterOptions.java
@@ -0,0 +1,282 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.config.GlobalConfiguration;
+import com.android.tradefed.config.IConfiguration;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.util.MultiMap;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/*
+ * A {@link IClusterOptions} implementation which contains cluster-related options.
+ */
+@OptionClass(alias = "cluster", global_namespace = false)
+public class ClusterOptions implements IClusterOptions {
+
+ /**
+ * The unique configuration object type name. Used to retrieve the singleton instance from the
+ * {@link GlobalConfiguration}.
+ *
+ * @see IConfiguration#getConfigurationObject(String)
+ */
+ public static final String TYPE_NAME = "cluster_options";
+
+ @Option(name = "service-url", description = "the base url of the tradefed cluster REST API")
+ public String mServiceUrl = null;
+
+ // TODO: The "service-account-keyfile" option should be HostOption (AOSP HostOptions.java).
+ @Option(
+ name = "service-account-keyfile",
+ description =
+ "The service account json key file. "
+ + "This is used by tradefed test scheduler (e.g. Tradefed Cluster) to "
+ + "authenticate the tradefed host. "
+ + "See google doc for the definition and how service account key works. "
+ + "https://cloud.google.com/iam/docs/service-accounts ")
+ private File mSchedulerServiceAccountKeyfile = null;
+
+ @Option(name = "cluster", description = "the cluster id for this TF instance", mandatory = true)
+ public String mClusterId = null;
+
+ @Option(
+ name = "next-cluster",
+ description =
+ "seconadary clusters for this TF instance to run commands from. If "
+ + "this option is set, TF will try to lease commands from these clusters in "
+ + "the order they are specified if it still has available devices after "
+ + "leasing commands from the primary cluster.")
+ public List<String> mNextClusterIds = new ArrayList<>();
+
+ @Option(name = "run-target-format", description = "the format for labelling run targets.")
+ private String mRunTargetFormat = null;
+
+ @Option(name = "disable-device-monitor", description = "disable Cluster device reporting")
+ private boolean mIsDeviceMonitorDisabled = false;
+
+ @Option(
+ name = "device-monitor-interval",
+ isTimeVal = true,
+ description = "the time interval between each device snapshot")
+ private long mDeviceMonitorSnapshotInterval = 60 * 1000;
+
+ @Option(
+ name = "device-group",
+ description =
+ "A multi-map from device group to device serials."
+ + " The key is a device group name and value is device serial.")
+ private MultiMap<String, String> mDeviceGroup = new MultiMap<String, String>();
+
+ @Option(
+ name = "device-tag",
+ description =
+ "A map for tagging device serials; each device may "
+ + "have one tag. This can be used for reporting in run-target")
+ private Map<String, String> mDeviceTag = new HashMap<>();
+
+ @Option(
+ name = "check-flashing-permits-on-lease",
+ description = "Check available flashing permits when leasing tasks")
+ private boolean mCheckFlashingPermitsOnLease = true;
+
+ @Option(
+ name = "invocation-heartbeat-interval",
+ isTimeVal = true,
+ description = "The time interval between invocation heartbeats")
+ private long mInvocationHeartbeatInterval = 5 * 60 * 1000;
+
+ @Option(name = "upload-invocation-status", description = "Upload invocation status to TFC")
+ private Boolean mShouldUploadInvocationStatus = false;
+
+ @Option(
+ name = "check-command-state",
+ description = "Check cluster command state to detect canceled invocations")
+ private boolean mCheckCommandState = false;
+
+ @Option(name = "connect-timeout", description = "HTTP connect timeout.", isTimeVal = true)
+ private int mConnectTimeout = 60000;
+
+ @Option(name = "read-timeout", description = "HTTP read timeout.", isTimeVal = true)
+ private int mReadTimeout = 60000;
+
+ /** {@inheritDoc} */
+ @Override
+ public String getServiceUrl() {
+ return mServiceUrl;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String getClusterId() {
+ return mClusterId;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public List<String> getNextClusterIds() {
+ return mNextClusterIds;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public MultiMap<String, String> getDeviceGroup() {
+ return mDeviceGroup;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Map<String, String> getDeviceTag() {
+ return mDeviceTag;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean checkFlashingPermitsOnLease() {
+ return mCheckFlashingPermitsOnLease;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String getRunTargetFormat() {
+ return mRunTargetFormat;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean isDeviceMonitorDisabled() {
+ return mIsDeviceMonitorDisabled;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getDeviceMonitorSnapshotInterval() {
+ return mDeviceMonitorSnapshotInterval;
+ }
+
+ /**
+ * Set the base url of the tradefed cluster REST API.
+ *
+ * <p>Exposed for testing.
+ */
+ void setServiceUrl(String url) {
+ mServiceUrl = url;
+ }
+
+ /**
+ * Set the cluster id for this TF instance.
+ *
+ * <p>Exposed for testing.
+ */
+ void setClusterId(String id) {
+ mClusterId = id;
+ }
+
+ /**
+ * Set the format for labelling run targets.
+ *
+ * <p>Exposed for testing.
+ */
+ void setRunTargetFormat(String format) {
+ mRunTargetFormat = format;
+ }
+
+ /**
+ * Set whether Cluster device reporting is disabled.
+ *
+ * <p>Exposed for testing.
+ */
+ void setDeviceMonitorDisabled(boolean disabled) {
+ mIsDeviceMonitorDisabled = disabled;
+ }
+
+ /**
+ * Set the time interval between each device snapshot in ms.
+ *
+ * <p>Exposed for testing.
+ */
+ void setDeviceMonitorSnapshotInterval(long interval) {
+ mDeviceMonitorSnapshotInterval = interval;
+ }
+
+ /**
+ * Set whether the scheduler should check if there are available flashing permits.
+ *
+ * <p>Exposed for testing.
+ */
+ void setCheckFlashingPermitsLease(boolean checkFlashingPermitsLease) {
+ mCheckFlashingPermitsOnLease = checkFlashingPermitsLease;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getInvocationHeartbeatInterval() {
+ return mInvocationHeartbeatInterval;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Boolean shouldUploadInvocationStatus() {
+ return mShouldUploadInvocationStatus;
+ }
+
+ /** Set the service account key file. */
+ @VisibleForTesting
+ void setSchedulerServiceAccountKeyfile(File keyFile) {
+ mSchedulerServiceAccountKeyfile = keyFile;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public File getSchedulerServiceAccountKeyfile() {
+ return mSchedulerServiceAccountKeyfile;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String getSchedulerServiceUrl() {
+ return mServiceUrl;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int getConnectTimeout() {
+ return mConnectTimeout;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int getReadTimeout() {
+ return mReadTimeout;
+ }
+
+ @Override
+ public boolean checkCommandState() {
+ return mCheckCommandState;
+ }
+
+ @VisibleForTesting
+ void setCheckCommandState(boolean checkCommandState) {
+ mCheckCommandState = checkCommandState;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/IClusterClient.java b/src/com/android/tradefed/cluster/IClusterClient.java
new file mode 100644
index 0000000..e803d7a
--- /dev/null
+++ b/src/com/android/tradefed/cluster/IClusterClient.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.config.GlobalConfiguration;
+import com.android.tradefed.config.IConfiguration;
+
+import org.json.JSONException;
+
+import java.io.IOException;
+import java.util.List;
+
+/** An interface for interacting with the TFC backend. */
+public interface IClusterClient {
+ /**
+ * The unique configuration object type name. Used to retrieve the singleton instance from the
+ * {@link GlobalConfiguration}.
+ *
+ * @see IConfiguration#getConfigurationObject(String)
+ */
+ public static final String TYPE_NAME = "cluster_client";
+
+ /**
+ * Get a {@link IClusterEventUploader} that can be used to upload {@link ClusterCommandEvent}s.
+ */
+ public IClusterEventUploader<ClusterCommandEvent> getCommandEventUploader();
+
+ /** Get a {@link IClusterEventUploader} that can be used to upload {@link ClusterHostEvent}s. */
+ public IClusterEventUploader<ClusterHostEvent> getHostEventUploader();
+
+ /**
+ * Lease {@link ClusterCommand} for the give host.
+ *
+ * @param clusterId cluster id for the host
+ * @param hostname hostname
+ * @param devices deviceInfos the host has
+ * @param nextClusterIds a list of next cluster IDs to lease commands from.
+ * @param maxTasksTolease the max number of tasks that can current be leased
+ * @return a list of {@link ClusterCommand}
+ * @throws JSONException
+ */
+ public List<ClusterCommand> leaseHostCommands(
+ final String clusterId,
+ final String hostname,
+ final List<ClusterDeviceInfo> devices,
+ final List<String> nextClusterIds,
+ final int maxTasksTolease)
+ throws JSONException;
+
+ /**
+ * Get {@link TestEnvironment} for a request.
+ *
+ * @param requestId
+ * @return a {@link TestEnvironment} object.
+ * @throws IOException
+ * @throws JSONException
+ */
+ public TestEnvironment getTestEnvironment(final String requestId)
+ throws IOException, JSONException;
+
+ /**
+ * Get {@link TestResource}s for a request.
+ *
+ * @param requestId
+ * @return a list of {@link TestResource}.
+ * @throws IOException
+ * @throws JSONException
+ */
+ public List<TestResource> getTestResources(final String requestId)
+ throws IOException, JSONException;
+
+ public TestContext getTestContext(final String requestId, final String commandId)
+ throws IOException, JSONException;
+
+ public void updateTestContext(
+ final String requestId, final String commandId, TestContext testContext)
+ throws IOException, JSONException;
+
+ /**
+ * Determine the state of a cluster command.
+ *
+ * @param requestId cluster request ID
+ * @param commandId cluster command ID
+ * @return cluster command's state, or {@link ClusterCommand.State#UNKNOWN} if state could not
+ * be determined
+ */
+ public ClusterCommand.State getCommandState(String requestId, String commandId);
+}
diff --git a/src/com/android/tradefed/cluster/IClusterEvent.java b/src/com/android/tradefed/cluster/IClusterEvent.java
new file mode 100644
index 0000000..33d07b6
--- /dev/null
+++ b/src/com/android/tradefed/cluster/IClusterEvent.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/** Interface for any cluster event to be uploaded to TFC. */
+public interface IClusterEvent {
+
+ public JSONObject toJSON() throws JSONException;
+}
diff --git a/src/com/android/tradefed/cluster/IClusterEventUploader.java b/src/com/android/tradefed/cluster/IClusterEventUploader.java
new file mode 100644
index 0000000..72ee6e7
--- /dev/null
+++ b/src/com/android/tradefed/cluster/IClusterEventUploader.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+/** Interface for ClusterEventUploader */
+public interface IClusterEventUploader<T extends IClusterEvent> {
+ /**
+ * Get the maximum number of events to upload at once.
+ *
+ * @param batchSize the maximum number of events to upload at once.
+ */
+ public void setMaxBatchSize(int batchSize);
+
+ /**
+ * Get the maximum batch size used when uploading events.
+ *
+ * @return the maximum batch size.
+ */
+ public int getMaxBatchSize();
+
+ /**
+ * Set how often we upload events to TFC.
+ *
+ * @param interval in ms for events to be uploaded to TFC.
+ */
+ public void setEventUploadInterval(long interval);
+
+ /**
+ * Get the upload interval.
+ *
+ * @return the upload interval in ms.
+ */
+ public long getEventUploadInterval();
+
+ /**
+ * Posts an event to TFC. This queues the event to be uploaded. Events will be batched and
+ * uploaded.
+ *
+ * @param event the event to upload
+ */
+ void postEvent(final T event);
+
+ /** Force an upload of all events queued. */
+ void flush();
+}
diff --git a/src/com/android/tradefed/cluster/IClusterOptions.java b/src/com/android/tradefed/cluster/IClusterOptions.java
new file mode 100644
index 0000000..c23f407
--- /dev/null
+++ b/src/com/android/tradefed/cluster/IClusterOptions.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.util.MultiMap;
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
+/** An interface for getting cluster-related options. */
+public interface IClusterOptions {
+
+ /** Get the base url of the tradefed cluster REST API. */
+ public String getServiceUrl();
+
+ /** Get the cluster id for this TF instance. */
+ public String getClusterId();
+
+ /** Get the secondary cluster ids for this TF instance. */
+ public List<String> getNextClusterIds();
+
+ /** Get the device group to device mapping. */
+ public MultiMap<String, String> getDeviceGroup();
+
+ /** Get the device serial to tag mapping. */
+ public Map<String, String> getDeviceTag();
+
+ /** Check if it should check for available flashing permits before leasing. */
+ boolean checkFlashingPermitsOnLease();
+
+ /** Get the format for labelling run targets. */
+ public String getRunTargetFormat();
+
+ /** Returns whether Cluster device reporting is disabled. */
+ public boolean isDeviceMonitorDisabled();
+
+ /** Get the time interval between each device snapshot in ms. */
+ public long getDeviceMonitorSnapshotInterval();
+
+ /** Returns whether TF should upload invocation status. */
+ public Boolean shouldUploadInvocationStatus();
+
+ /** Get the time interval between invocation heartbeats in ms. */
+ public long getInvocationHeartbeatInterval();
+
+ /** Get http connect timeout. */
+ public int getConnectTimeout();
+
+ /** Get http read timeout. */
+ public int getReadTimeout();
+
+ /** Get the tradefed test scheduler service account key file. */
+ public File getSchedulerServiceAccountKeyfile();
+
+ /** Get the tradefed test scheduler service URL. */
+ public String getSchedulerServiceUrl();
+
+ /** Whether the command state (on the TF cluster) should be checked during heartbeat. */
+ public boolean checkCommandState();
+}
diff --git a/src/com/android/tradefed/cluster/InvocationStatus.java b/src/com/android/tradefed/cluster/InvocationStatus.java
new file mode 100644
index 0000000..34bcfdc
--- /dev/null
+++ b/src/com/android/tradefed/cluster/InvocationStatus.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A class to store invocation status.
+ *
+ * <p>This also includes statuses of test groups that run within the invocation.
+ */
+public class InvocationStatus {
+
+ private List<TestGroupStatus> mTestGroupStatuses = new ArrayList<TestGroupStatus>();
+
+ public void addTestGroupStatus(final TestGroupStatus progress) {
+ mTestGroupStatuses.add(progress);
+ }
+
+ public List<TestGroupStatus> getTestGroupStatuses() {
+ return Collections.unmodifiableList(mTestGroupStatuses);
+ }
+
+ public JSONObject toJSON() throws JSONException {
+ final JSONObject json = new JSONObject();
+ final JSONArray testModuleStatuses = new JSONArray();
+ for (final TestGroupStatus o : mTestGroupStatuses) {
+ testModuleStatuses.put(o.toJSON());
+ }
+ json.put("test_group_statuses", testModuleStatuses);
+ return json;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/SubprocessConfigBuilder.java b/src/com/android/tradefed/cluster/SubprocessConfigBuilder.java
new file mode 100644
index 0000000..2b32638
--- /dev/null
+++ b/src/com/android/tradefed/cluster/SubprocessConfigBuilder.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.config.Configuration;
+import com.android.tradefed.config.ConfigurationUtil;
+import com.android.tradefed.result.LegacySubprocessResultsReporter;
+
+import org.kxml2.io.KXmlSerializer;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+/**
+ * Build a wrapper TF config XML for an existing TF config.
+ *
+ * <p>A wrapper XML allows to enable subprocess reporting on an existing TF config.
+ */
+public class SubprocessConfigBuilder {
+ private static final String INCLUDE_NAME = "include";
+ private static final String REPORTER_CLASS = LegacySubprocessResultsReporter.class.getName();
+ private static final String OPTION_KEY = "subprocess-report-port";
+ private static final String CONFIG_DESCRIPTION = "Cluster Command Launcher config";
+
+ private File mWorkdir;
+
+ private String mOriginalConfig;
+
+ private String mPort;
+
+ public SubprocessConfigBuilder setWorkingDir(File dir) {
+ mWorkdir = dir;
+ return this;
+ }
+
+ public SubprocessConfigBuilder setOriginalConfig(String config) {
+ mOriginalConfig = config;
+ return this;
+ }
+
+ public SubprocessConfigBuilder setPort(String port) {
+ mPort = port;
+ return this;
+ }
+
+ public File build() throws IOException {
+ // Make a new config name based on the original config name to make it possible to find
+ // out the original command line from a modified one.
+ // FIXME: Find a better way to preserve the original command line.
+ String configName = mOriginalConfig.replace("/", "$") + ".xml";
+ // mOriginalConfig is from another test suite, so its content is hard to know at this
+ // time. So it doesn't load mOriginalConfig as IConfiguration and add additional config.
+ // Instead, it creates a wrapper config including mOriginalConfig.
+ File f = new File(mWorkdir, configName);
+ PrintWriter writer = new PrintWriter((f));
+ KXmlSerializer serializer = new KXmlSerializer();
+ serializer.setOutput(writer);
+ serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
+ serializer.startDocument("UTF-8", null);
+ serializer.startTag(null, ConfigurationUtil.CONFIGURATION_NAME);
+ serializer.attribute(
+ null, Configuration.CONFIGURATION_DESCRIPTION_TYPE_NAME, CONFIG_DESCRIPTION);
+
+ serializer.startTag(null, INCLUDE_NAME);
+ serializer.attribute(null, ConfigurationUtil.NAME_NAME, mOriginalConfig);
+ serializer.endTag(null, INCLUDE_NAME);
+
+ if (mPort != null) {
+ serializer.startTag(null, Configuration.RESULT_REPORTER_TYPE_NAME);
+ serializer.attribute(null, ConfigurationUtil.CLASS_NAME, REPORTER_CLASS);
+
+ serializer.startTag(null, ConfigurationUtil.OPTION_NAME);
+ serializer.attribute(null, ConfigurationUtil.NAME_NAME, OPTION_KEY);
+ serializer.attribute(null, ConfigurationUtil.VALUE_NAME, mPort);
+ serializer.endTag(null, ConfigurationUtil.OPTION_NAME);
+
+ serializer.endTag(null, Configuration.RESULT_REPORTER_TYPE_NAME);
+ }
+
+ serializer.endTag(null, ConfigurationUtil.CONFIGURATION_NAME);
+ serializer.endDocument();
+
+ writer.close();
+ return f;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/SubprocessReportingHelper.java b/src/com/android/tradefed/cluster/SubprocessReportingHelper.java
new file mode 100644
index 0000000..d48522e
--- /dev/null
+++ b/src/com/android/tradefed/cluster/SubprocessReportingHelper.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.log.LogUtil;
+import com.android.tradefed.result.LegacySubprocessResultsReporter;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.QuotationAwareTokenizer;
+import com.android.tradefed.util.StreamUtil;
+import com.android.tradefed.util.ZipUtil2;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Set;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+/**
+ * A class to build a wrapper configuration file to use subprocess results reporter for a cluster
+ * command.
+ */
+public class SubprocessReportingHelper {
+ private static final String REPORTER_JAR_NAME = "subprocess-results-reporter.jar";
+ private static final String CLASS_FILTER =
+ String.format(
+ "(^%s|^%s|^%s|^%s|^%s).*class$",
+ "LegacySubprocessResultsReporter",
+ "SubprocessTestResultsParser",
+ "SubprocessEventHelper",
+ "SubprocessResultsReporter",
+ "ISupportGranularResults");
+
+ /**
+ * Dynamically generate extract .class file from tradefed.jar and generate new subprocess
+ * results reporter jar.
+ *
+ * @param parentDir parent directory of subprocess results reporter jar.
+ * @return subprocess result reporter jar.
+ * @throws IOException
+ */
+ public File createSubprocessReporterJar(File parentDir) throws IOException {
+ File reporterJar = new File(parentDir, REPORTER_JAR_NAME);
+ File tfJar =
+ new File(
+ LegacySubprocessResultsReporter.class
+ .getProtectionDomain()
+ .getCodeSource()
+ .getLocation()
+ .getPath());
+ // tfJar is directory of .class file when running JUnit test from Eclipse IDE
+ if (tfJar.isDirectory()) {
+ Set<File> classFiles = FileUtil.findFilesObject(tfJar, CLASS_FILTER);
+ Manifest manifest = new Manifest();
+ createJar(reporterJar, manifest, classFiles);
+ }
+ // tfJar is the tradefed.jar when running with tradefed.
+ else {
+ File extractedJar = ZipUtil2.extractZipToTemp(tfJar, "tmp-jar");
+ try {
+ Set<File> classFiles = FileUtil.findFilesObject(extractedJar, CLASS_FILTER);
+ File mf = FileUtil.findFile(extractedJar, "MANIFEST.MF");
+ Manifest manifest = new Manifest(new FileInputStream(mf));
+ createJar(reporterJar, manifest, classFiles);
+ } finally {
+ FileUtil.recursiveDelete(extractedJar);
+ }
+ }
+ return reporterJar;
+ }
+
+ /**
+ * Create jar file.
+ *
+ * @param jar jar file to be created.
+ * @param manifest manifest file.
+ * @throws IOException
+ */
+ private void createJar(File jar, Manifest manifest, Set<File> classFiles) throws IOException {
+ try (JarOutputStream jarOutput = new JarOutputStream(new FileOutputStream(jar), manifest)) {
+ for (File file : classFiles) {
+ try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(file))) {
+ String path = file.getPath();
+ JarEntry entry = new JarEntry(path.substring(path.indexOf("com")));
+ entry.setTime(file.lastModified());
+ jarOutput.putNextEntry(entry);
+ StreamUtil.copyStreams(in, jarOutput);
+ jarOutput.closeEntry();
+ }
+ }
+ }
+ }
+
+ /**
+ * Get a new command line whose configuration argument is replaced by a newly-created wrapper
+ * configuration.
+ *
+ * <p>The resulting command line will reference a generate XML file in parentDir and needs to
+ * run from parentDir.
+ *
+ * @param commandLine old command line that will be run by subprocess.
+ * @param port port number that subprocess should use to report results.
+ * @param parentDir parent directory of new wrapper configuration.
+ * @return new command line, whose first argument is wrapper config.
+ * @throws IOException
+ */
+ public String buildNewCommandConfig(String commandLine, String port, File parentDir)
+ throws IOException {
+ String[] tokens = QuotationAwareTokenizer.tokenizeLine(commandLine);
+ SubprocessConfigBuilder builder = new SubprocessConfigBuilder();
+ builder.setWorkingDir(parentDir).setOriginalConfig(tokens[0]).setPort(port);
+ File f = builder.build();
+ LogUtil.CLog.i("Generating new configuration:\n %s", FileUtil.readStringFromFile(f));
+ tokens[0] = f.getName();
+ return QuotationAwareTokenizer.combineTokens(tokens);
+ }
+}
diff --git a/src/com/android/tradefed/cluster/TestContext.java b/src/com/android/tradefed/cluster/TestContext.java
new file mode 100644
index 0000000..2b9d38b
--- /dev/null
+++ b/src/com/android/tradefed/cluster/TestContext.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.log.LogUtil.CLog;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/**
+ * A class to model a TestContext message of TFC API.
+ *
+ * <p>A TestContext message is used to store and retrieve contextual information being passed across
+ * multiple attempts of same test command.
+ */
+public class TestContext {
+
+ String mCommandLine = null;
+ final Map<String, String> mEnvVars = new TreeMap<>();
+ final List<TestResource> mTestResources = new ArrayList<>();
+
+ public String getCommandLine() {
+ return mCommandLine;
+ }
+
+ public void setCommandLine(String commandLine) {
+ mCommandLine = commandLine;
+ }
+
+ public Map<String, String> getEnvVars() {
+ return Collections.unmodifiableMap(mEnvVars);
+ }
+
+ public void addEnvVars(Map<String, String> envVars) {
+ mEnvVars.putAll(envVars);
+ }
+
+ public List<TestResource> getTestResources() {
+ return Collections.unmodifiableList(mTestResources);
+ }
+
+ public void addTestResource(TestResource testResource) {
+ mTestResources.add(testResource);
+ }
+
+ public static TestContext fromJson(JSONObject json) throws JSONException {
+ final TestContext obj = new TestContext();
+ obj.mCommandLine = json.optString("command_line");
+ final JSONArray envVars = json.optJSONArray("env_vars");
+ if (envVars != null) {
+ for (int i = 0; i < envVars.length(); i++) {
+ final JSONObject pair = envVars.getJSONObject(i);
+ obj.mEnvVars.put(pair.getString("key"), pair.getString("value"));
+ }
+ }
+ final JSONArray testResources = json.optJSONArray("test_resources");
+ if (testResources != null) {
+ obj.mTestResources.addAll(TestResource.fromJsonArray(testResources));
+ }
+ return obj;
+ }
+
+ public JSONObject toJson() throws JSONException {
+ final JSONObject json = new JSONObject();
+ json.put("command_line", mCommandLine);
+ final JSONArray envVars = new JSONArray();
+ for (Map.Entry<String, String> entry : mEnvVars.entrySet()) {
+ final JSONObject pair = new JSONObject();
+ pair.put("key", entry.getKey());
+ pair.put("value", entry.getValue());
+ envVars.put(pair);
+ }
+ json.put("env_vars", envVars);
+ final JSONArray testResources = new JSONArray();
+ for (TestResource obj : mTestResources) {
+ testResources.put(obj.toJson());
+ }
+ json.put("test_resources", testResources);
+ return json;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return toJson().toString();
+ } catch (JSONException e) {
+ CLog.w("Failed to convert to JSON: %s", e);
+ }
+ return null;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof TestContext)) {
+ return false;
+ }
+
+ return toString().equals(o.toString());
+ }
+}
diff --git a/src/com/android/tradefed/cluster/TestEnvironment.java b/src/com/android/tradefed/cluster/TestEnvironment.java
new file mode 100644
index 0000000..9b9afb9
--- /dev/null
+++ b/src/com/android/tradefed/cluster/TestEnvironment.java
@@ -0,0 +1,256 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.annotations.VisibleForTesting;
+import com.android.tradefed.log.LogUtil.CLog;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/** A class to model a TestEnvironment message returned by TFC API. */
+public class TestEnvironment {
+
+ final Map<String, String> mEnvVars = new HashMap<>();
+ final List<String> mSetupScripts = new ArrayList<>();
+ final List<String> mOutputFilePatterns = new ArrayList<>();
+ String mOutputFileUploadUrl = null;
+ boolean mUseSubprocessReporting = false;
+ long mOutputIdleTimeout = 0L;
+ final Map<String, String> mJavaProperties = new HashMap<>();
+ String mContextFilePattern = null;
+ private final List<String> mExtraContextFiles = new ArrayList<>();
+ String mRetryCommandLine = null;
+ String mLogLevel = null;
+ final List<TradefedConfigObject> mTradefedConfigObjects = new ArrayList<>();
+
+ /**
+ * Adds an environment variable.
+ *
+ * @param name a variable name.
+ * @param value a variable value.
+ */
+ public void addEnvVar(final String name, final String value) {
+ mEnvVars.put(name, value);
+ }
+
+ /**
+ * Returns a {@link Map} object containing all env vars.
+ *
+ * @return unmodifiable map of all env vars.
+ */
+ public Map<String, String> getEnvVars() {
+ return Collections.unmodifiableMap(mEnvVars);
+ }
+
+ /**
+ * Adds a setup script command.
+ *
+ * @param s a setup script command.
+ */
+ public void addSetupScripts(final String s) {
+ mSetupScripts.add(s);
+ }
+
+ /**
+ * Returns a list of setup script commands.
+ *
+ * @return unmodifiable list of commands
+ */
+ public List<String> getSetupScripts() {
+ return Collections.unmodifiableList(mSetupScripts);
+ }
+
+ /**
+ * Adds an output file pattern.
+ *
+ * @param s a file pattern.
+ */
+ public void addOutputFilePattern(final String s) {
+ mOutputFilePatterns.add(s);
+ }
+
+ /**
+ * Returns a list of output file patterns.
+ *
+ * @return unmodifiable list of file patterns.
+ */
+ public List<String> getOutputFilePatterns() {
+ return Collections.unmodifiableList(mOutputFilePatterns);
+ }
+
+ /**
+ * Sets an output file upload URL.
+ *
+ * @param s a URL.
+ */
+ public void setOutputFileUploadUrl(final String s) {
+ mOutputFileUploadUrl = s;
+ }
+
+ /**
+ * Returns an output file upload URL.
+ *
+ * @return a URL.
+ */
+ public String getOutputFileUploadUrl() {
+ return mOutputFileUploadUrl;
+ }
+
+ /**
+ * Returns whether to use subprocess reporting.
+ *
+ * @return a boolean.
+ */
+ public boolean useSubprocessReporting() {
+ return mUseSubprocessReporting;
+ }
+
+ public void setUseSubprocessReporting(boolean f) {
+ mUseSubprocessReporting = f;
+ }
+
+ /** @return maximum millis to wait for an idle subprocess */
+ public long getOutputIdleTimeout() {
+ return mOutputIdleTimeout;
+ }
+
+ public void setOutputIdleTimeout(long outputIdleTimeout) {
+ mOutputIdleTimeout = outputIdleTimeout;
+ }
+
+ /**
+ * Adds a java property.
+ *
+ * @param name a property name.
+ * @param value a property value.
+ */
+ public void addJavaProperty(final String name, final String value) {
+ mJavaProperties.put(name, value);
+ }
+
+ /**
+ * Returns a {@link Map} object containing all Java properties.
+ *
+ * @return unmodifiable map of all runner properties.
+ */
+ public Map<String, String> getJavaProperties() {
+ return Collections.unmodifiableMap(mJavaProperties);
+ }
+
+ public String getContextFilePattern() {
+ return mContextFilePattern;
+ }
+
+ /** Adds a file path to append to the context file. */
+ public void addExtraContextFile(String path) {
+ mExtraContextFiles.add(path);
+ }
+
+ /** @return list of additional file paths to append to context file */
+ public List<String> getExtraContextFiles() {
+ return Collections.unmodifiableList(mExtraContextFiles);
+ }
+
+ public String getRetryCommandLine() {
+ return mRetryCommandLine;
+ }
+
+ public String getLogLevel() {
+ return mLogLevel;
+ }
+
+ public List<TradefedConfigObject> getTradefedConfigObjects() {
+ return Collections.unmodifiableList(mTradefedConfigObjects);
+ }
+
+ /**
+ * Adds a {@link TradefedConfigObject}.
+ *
+ * @param obj a {@link TradefedConfigObject}.
+ */
+ @VisibleForTesting
+ void addTradefedConfigObject(TradefedConfigObject obj) {
+ mTradefedConfigObjects.add(obj);
+ }
+
+ public static TestEnvironment fromJson(JSONObject json) throws JSONException {
+ TestEnvironment obj = new TestEnvironment();
+ final JSONArray envVars = json.optJSONArray("env_vars");
+ if (envVars != null) {
+ for (int i = 0; i < envVars.length(); i++) {
+ final JSONObject envVar = envVars.getJSONObject(i);
+ obj.addEnvVar(envVar.getString("key"), envVar.getString("value"));
+ }
+ } else {
+ CLog.w("env_vars is null");
+ }
+ final JSONArray javaProperties = json.optJSONArray("java_properties");
+ if (javaProperties != null) {
+ for (int i = 0; i < javaProperties.length(); i++) {
+ final JSONObject javaProperty = javaProperties.getJSONObject(i);
+ obj.addJavaProperty(javaProperty.getString("key"), javaProperty.getString("value"));
+ }
+ } else {
+ CLog.w("java_properties is null");
+ }
+ final JSONArray scripts = json.optJSONArray("setup_scripts");
+ if (scripts != null) {
+ for (int i = 0; i < scripts.length(); i++) {
+ obj.addSetupScripts(scripts.getString(i));
+ }
+ } else {
+ CLog.w("setup_scripts is null");
+ }
+ final JSONArray patterns = json.optJSONArray("output_file_patterns");
+ if (patterns != null) {
+ for (int i = 0; i < patterns.length(); i++) {
+ obj.addOutputFilePattern(patterns.getString(i));
+ }
+ } else {
+ CLog.w("output_file_patterns is null");
+ }
+ final String url = json.optString("output_file_upload_url");
+ if (url != null) {
+ obj.setOutputFileUploadUrl(url);
+ } else {
+ CLog.w("output_file_upload_url is null");
+ }
+ obj.mUseSubprocessReporting = json.optBoolean("use_subprocess_reporting");
+ obj.mOutputIdleTimeout = json.optLong("output_idle_timeout_millis", 0L);
+ obj.mContextFilePattern = json.optString("context_file_pattern");
+ JSONArray extraContextFiles = json.optJSONArray("extra_context_files");
+ if (extraContextFiles != null) {
+ for (int i = 0; i < extraContextFiles.length(); i++) {
+ obj.addExtraContextFile(extraContextFiles.getString(i));
+ }
+ } else {
+ CLog.w("extra_context_files is null");
+ }
+ obj.mRetryCommandLine = json.optString("retry_command_line");
+ obj.mLogLevel = json.optString("log_level");
+ final JSONArray arr = json.optJSONArray("tradefed_config_objects");
+ if (arr != null) {
+ obj.mTradefedConfigObjects.addAll(TradefedConfigObject.fromJsonArray(arr));
+ }
+ return obj;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/TestGroupStatus.java b/src/com/android/tradefed/cluster/TestGroupStatus.java
new file mode 100644
index 0000000..3497f06
--- /dev/null
+++ b/src/com/android/tradefed/cluster/TestGroupStatus.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/**
+ * A class to store status of a test group.
+ *
+ * <p>A test group corresponds to a group of test cases reported under a same test run name.
+ */
+public class TestGroupStatus {
+
+ private String mName;
+ private int mTotalTestCount;
+ private int mCompletedTestCount;
+ private int mFailedTestCount;
+ private boolean mIsComplete;
+ private long mElapsedTime;
+ private String mFailureMessage;
+
+ public TestGroupStatus(
+ final String name,
+ final int totalTestCount,
+ final int completedTestCount,
+ final int failedTestCount,
+ final boolean isComplete,
+ final long elapsedTime,
+ final String failureMessage) {
+ mName = name;
+ mTotalTestCount = totalTestCount;
+ mCompletedTestCount = completedTestCount;
+ mFailedTestCount = failedTestCount;
+ mIsComplete = isComplete;
+ mElapsedTime = elapsedTime;
+ mFailureMessage = failureMessage;
+ }
+
+ public String getName() {
+ return mName;
+ }
+
+ public int getTotalTestCount() {
+ return mTotalTestCount;
+ }
+
+ public int getCompletedTestCount() {
+ return mCompletedTestCount;
+ }
+
+ public int getFailedTestCount() {
+ return mFailedTestCount;
+ }
+
+ public boolean isComplete() {
+ return mIsComplete;
+ }
+
+ public long getElapsedTime() {
+ return mElapsedTime;
+ }
+
+ public String getFailureMessage() {
+ return mFailureMessage;
+ }
+
+ public JSONObject toJSON() throws JSONException {
+ final JSONObject json = new JSONObject();
+ json.put("name", mName);
+ json.put("total_test_count", mTotalTestCount);
+ json.put("completed_test_count", mCompletedTestCount);
+ json.put("failed_test_count", mFailedTestCount);
+ json.put("is_complete", mIsComplete);
+ json.put("elapsed_time", mElapsedTime);
+ json.put("failure_message", mFailureMessage);
+ return json;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/TestOutputUploader.java b/src/com/android/tradefed/cluster/TestOutputUploader.java
new file mode 100644
index 0000000..c00f1c3
--- /dev/null
+++ b/src/com/android/tradefed/cluster/TestOutputUploader.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.loganalysis.util.ArrayUtil;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.util.CommandResult;
+import com.android.tradefed.util.CommandStatus;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.IRunUtil;
+import com.android.tradefed.util.RunUtil;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.List;
+
+/** A class to upload test output files to GCS/HTTP. */
+public class TestOutputUploader {
+
+ static final long UPLOAD_TIMEOUT_MS = 30 * 60 * 1000;
+ static final long RETRY_INTERVAL_MS = 10 * 1000;
+ static final int MAX_RETRY_COUNT = 2;
+ static final String FILE_PROTOCOL = "file";
+ static final String GCS_PROTOCOL = "gs";
+ static final String HTTP_PROTOCOL = "http";
+ static final String HTTPS_PROTOCOL = "https";
+
+ private String mUploadUrl = null;
+ private String mProtocol = null;
+ private IRunUtil mRunUtil = null;
+
+ public void setUploadUrl(final String url) throws MalformedURLException {
+ mUploadUrl = url;
+ URL urlObj = new URL(url);
+ mProtocol = urlObj.getProtocol();
+ }
+
+ public String uploadFile(File file, final String destinationPath) throws IOException {
+ String uploadUrl = mUploadUrl;
+ if (!mUploadUrl.endsWith("/")) {
+ uploadUrl += "/";
+ }
+ if (destinationPath != null) {
+ uploadUrl += destinationPath;
+ }
+ CLog.i("Uploading %s to %s", file.getAbsolutePath(), uploadUrl);
+ if (FILE_PROTOCOL.equals(mProtocol)) {
+ final File dir = new File(new URL(uploadUrl).getPath());
+ if (!dir.exists()) {
+ dir.mkdirs();
+ }
+ final File destFile = new File(dir, file.getName());
+ FileUtil.copyFile(file, destFile);
+ } else {
+ final List<String> cmdArgs = buildUploadCommandArgs(file, uploadUrl);
+ final CommandResult result =
+ getRunUtil()
+ .runTimedCmdRetry(
+ UPLOAD_TIMEOUT_MS,
+ RETRY_INTERVAL_MS,
+ MAX_RETRY_COUNT,
+ cmdArgs.toArray(new String[0]));
+ if (!result.getStatus().equals(CommandStatus.SUCCESS)) {
+ final String msg =
+ String.format(
+ "failed to upload %s: command status=%s",
+ file.getAbsolutePath(), result.getStatus());
+ CLog.e(msg);
+ CLog.e("stdout:\n'''\n%s'''\n", result.getStdout());
+ CLog.d("stderr:\n'''\n%s'''\n", result.getStderr());
+ throw new RuntimeException(msg);
+ }
+ }
+ String baseUrl = uploadUrl;
+ if (!baseUrl.endsWith("/")) {
+ baseUrl += "/";
+ }
+ return baseUrl + file.getName();
+ }
+
+ private List<String> buildUploadCommandArgs(File file, String uploadUrl) {
+ if (mUploadUrl == null) {
+ throw new IllegalStateException("upload url is not set");
+ }
+ switch (mProtocol) {
+ case GCS_PROTOCOL:
+ return ArrayUtil.list("gsutil", "cp", file.getAbsolutePath(), uploadUrl);
+ case HTTP_PROTOCOL:
+ case HTTPS_PROTOCOL:
+ // Add -L option to handle redirect.
+ return ArrayUtil.list(
+ "curl",
+ "-X",
+ "POST",
+ "-F file=@" + file.getAbsolutePath(),
+ "-fL",
+ uploadUrl);
+ }
+ throw new IllegalArgumentException(
+ String.format("Protocol '%s' is not supported", mProtocol));
+ }
+
+ @VisibleForTesting
+ IRunUtil getRunUtil() {
+ if (mRunUtil == null) {
+ mRunUtil = RunUtil.getDefault();
+ }
+ return mRunUtil;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/TestResource.java b/src/com/android/tradefed/cluster/TestResource.java
new file mode 100644
index 0000000..75f209b
--- /dev/null
+++ b/src/com/android/tradefed/cluster/TestResource.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/** A class to model a TestResource message returned by TFC API. */
+public class TestResource {
+
+ private final String mName;
+ private final String mUrl;
+
+ TestResource(final String name, final String url) {
+ mName = name;
+ mUrl = url;
+ }
+
+ public String getName() {
+ return mName;
+ }
+
+ public String getUrl() {
+ return mUrl;
+ }
+
+ public JSONObject toJson() throws JSONException {
+ final JSONObject json = new JSONObject();
+ json.put("name", mName);
+ json.put("url", mUrl);
+ return json;
+ }
+
+ public static TestResource fromJson(JSONObject json) {
+ return new TestResource(json.optString("name"), json.optString("url"));
+ }
+
+ public static List<TestResource> fromJsonArray(JSONArray jsonArray) throws JSONException {
+ final List<TestResource> objs = new ArrayList<>();
+ for (int i = 0; i < jsonArray.length(); i++) {
+ objs.add(TestResource.fromJson(jsonArray.getJSONObject(i)));
+ }
+ return objs;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/TestResourceDownloader.java b/src/com/android/tradefed/cluster/TestResourceDownloader.java
new file mode 100644
index 0000000..5e485d7
--- /dev/null
+++ b/src/com/android/tradefed/cluster/TestResourceDownloader.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.loganalysis.util.ArrayUtil;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.util.CommandResult;
+import com.android.tradefed.util.CommandStatus;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.IRunUtil;
+import com.android.tradefed.util.RunUtil;
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.List;
+
+/** A class to download test resource files from file system/GCS/HTTP. */
+public class TestResourceDownloader {
+
+ private static final long DOWNLOAD_TIMEOUT_MS = 30 * 60 * 1000;
+ private static final long RETRY_INTERVAL_MS = 10 * 1000;
+ private static final int MAX_RETRY_COUNT = 2;
+
+ private final File mRootDir;
+ private IRunUtil mRunUtil = null;
+
+ public TestResourceDownloader(final File rootDir) {
+ mRootDir = rootDir;
+ }
+
+ public File download(TestResource resource) throws IOException {
+ final URL url = new URL(resource.getUrl());
+ final String protocol = url.getProtocol();
+ final File dest = new File(mRootDir, resource.getName());
+ final File parent = dest.getParentFile();
+ if (!parent.exists()) {
+ parent.mkdirs();
+ }
+ if ("file".equals(protocol)) {
+ final File src = new File(resource.getUrl().substring(6));
+ FileUtil.hardlinkFile(src, dest);
+ return dest;
+ }
+
+ final List<String> cmdArgs = buildDownloadCommandArgs(url, dest);
+ final CommandResult result =
+ getRunUtil()
+ .runTimedCmdRetry(
+ DOWNLOAD_TIMEOUT_MS,
+ RETRY_INTERVAL_MS,
+ MAX_RETRY_COUNT + 1,
+ cmdArgs.toArray(new String[0]));
+ if (!result.getStatus().equals(CommandStatus.SUCCESS)) {
+ final String msg =
+ String.format(
+ "Failed to download %s: command status=%s", url, result.getStatus());
+ CLog.e(msg);
+ CLog.e("stdout:\n'''\n%s'''\n", result.getStdout());
+ CLog.d("stderr:\n'''\n%s'''\n", result.getStderr());
+ throw new RuntimeException(msg);
+ }
+ return dest;
+ }
+
+ /** Build a list of command line arguments to download a file. */
+ private List<String> buildDownloadCommandArgs(URL url, File file) {
+ final String protocol = url.getProtocol();
+ if ("gs".equals(protocol)) {
+ // FIXME: Check whether gsutil is available on a host.
+ return ArrayUtil.list("gsutil", "cp", url.toString(), file.getAbsolutePath());
+ }
+ if ("http".equals(protocol) || "https".equals(protocol)) {
+ // FIXME: Check whether curl is available on a host.
+ // Add -L option to handle redirect.
+ return ArrayUtil.list("curl", "-o", file.getAbsolutePath(), "-fL", url.toString());
+ }
+ throw new UnsupportedOperationException("protocol " + protocol + " is not supported");
+ }
+
+ IRunUtil getRunUtil() {
+ if (mRunUtil == null) {
+ mRunUtil = new RunUtil();
+ }
+ return mRunUtil;
+ }
+}
diff --git a/src/com/android/tradefed/cluster/TradefedConfigObject.java b/src/com/android/tradefed/cluster/TradefedConfigObject.java
new file mode 100644
index 0000000..ffc5475
--- /dev/null
+++ b/src/com/android/tradefed/cluster/TradefedConfigObject.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.config.Configuration;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.targetprep.ITargetPreparer;
+import com.android.tradefed.util.MultiMap;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A class to model a TradefedConfigObject message of TFC API.
+ *
+ * <p>A TradefedConfigObject message is a part of a TestEnvironment message and used to pass down
+ * extra {@link ITargetPreparer} or {@link ITestInvocationListener} to be added to a cluster command
+ * launcher config.
+ */
+public class TradefedConfigObject {
+
+ /**
+ * A list of configuration object types which can be injected to a cluster command config.
+ *
+ * <p>This must be a subset of configuration object types defined in {@link Configuration}.
+ */
+ public static enum Type {
+ UNKNOWN,
+ TARGET_PREPARER,
+ RESULT_REPORTER
+ }
+
+ private final Type mType;
+ private final String mClassName;
+ private final MultiMap<String, String> mOptionValues;
+
+ TradefedConfigObject(Type type, String className, MultiMap<String, String> optionValues) {
+ mType = type;
+ mClassName = className;
+ mOptionValues = optionValues;
+ }
+
+ public Type getType() {
+ return mType;
+ }
+
+ public String getClassName() {
+ return mClassName;
+ }
+
+ public MultiMap<String, String> getOptionValues() {
+ return mOptionValues;
+ }
+
+ public static TradefedConfigObject fromJson(JSONObject json) throws JSONException {
+ MultiMap<String, String> optionValues = new MultiMap<>();
+ JSONArray arr = json.optJSONArray("option_values");
+ if (arr != null) {
+ for (int i = 0; i < arr.length(); i++) {
+ JSONObject pair = arr.getJSONObject(i);
+ String key = pair.getString("key");
+ JSONArray valueArr = pair.optJSONArray("values");
+ if (valueArr == null) {
+ optionValues.put(key, null);
+ } else {
+ for (int j = 0; j < valueArr.length(); j++) {
+ optionValues.put(key, valueArr.getString(j));
+ }
+ }
+ }
+ }
+ Type type = Type.valueOf(json.optString("type", Type.UNKNOWN.name()));
+ return new TradefedConfigObject(type, json.getString("class_name"), optionValues);
+ }
+
+ public static List<TradefedConfigObject> fromJsonArray(JSONArray arr) throws JSONException {
+ List<TradefedConfigObject> objs = new ArrayList<>();
+ for (int i = 0; i < arr.length(); i++) {
+ objs.add(fromJson(arr.getJSONObject(i)));
+ }
+ return objs;
+ }
+}
diff --git a/src/com/android/tradefed/util/IRestApiHelper.java b/src/com/android/tradefed/util/IRestApiHelper.java
new file mode 100644
index 0000000..12d2b07
--- /dev/null
+++ b/src/com/android/tradefed/util/IRestApiHelper.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.util;
+
+import com.google.api.client.http.HttpResponse;
+
+import org.json.JSONObject;
+
+import java.io.IOException;
+import java.util.Map;
+
+/** A helper interface for performing REST API calls. */
+public interface IRestApiHelper {
+
+ /**
+ * Executes an API request.
+ *
+ * @param method a HTTP method of the request
+ * @param uriParts URL encoded URI parts to be used to construct the request URI.
+ * @param options unencoded parameter names and values used to construct the query string
+ * @param data data to be sent with the request
+ * @return a HttpResponse object
+ * @throws IOException
+ */
+ public HttpResponse execute(
+ String method, String[] uriParts, Map<String, Object> options, JSONObject data)
+ throws IOException;
+}
diff --git a/src/com/android/tradefed/util/RestApiHelper.java b/src/com/android/tradefed/util/RestApiHelper.java
new file mode 100644
index 0000000..7970dc4
--- /dev/null
+++ b/src/com/android/tradefed/util/RestApiHelper.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.util;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.ByteArrayContent;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpContent;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpRequestFactory;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.JsonObjectParser;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.json.JSONObject;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.Collection;
+import java.util.Map;
+
+/** A helper class for performing REST API calls. */
+public class RestApiHelper implements IRestApiHelper {
+
+ protected static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
+ protected static final String JSON_MIME = "application/json";
+
+ private HttpRequestFactory mRequestFactory;
+ private String mBaseUri;
+
+ /**
+ * Creates an API helper instance with the given information.
+ *
+ * @param baseUri the base URI of API
+ * @param requestFactory the factory to use when creating {@link HttpRequest}s.
+ */
+ public RestApiHelper(HttpRequestFactory requestFactory, String baseUri) {
+ mRequestFactory = requestFactory;
+ // Make sure the uri ends with a slash to avoid GenericUrl weirdness later
+ mBaseUri = baseUri.endsWith("/") ? baseUri : baseUri + "/";
+ }
+
+ /**
+ * Creates an API helper instance which uses a {@link GoogleCredential} for authentication.
+ *
+ * @param baseUri the base URI of the API
+ * @param serviceAccount the name of the service account to use
+ * @param keyFile the service account key file
+ * @param scopes the collection of OAuth scopes to use with the service account
+ * @throws GeneralSecurityException
+ * @throws IOException
+ */
+ @Deprecated
+ public static RestApiHelper newInstanceWithGoogleCredential(
+ String baseUri, String serviceAccount, File keyFile, Collection<String> scopes)
+ throws GeneralSecurityException, IOException {
+
+ HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
+ GoogleCredential credential =
+ GoogleApiClientUtil.createCredentialFromP12File(serviceAccount, keyFile, scopes);
+ HttpRequestFactory requestFactory = transport.createRequestFactory(credential);
+
+ return new RestApiHelper(requestFactory, baseUri);
+ }
+
+ /**
+ * Creates an API helper instance which uses a {@link GoogleCredential} for authentication.
+ *
+ * @param baseUri the base URI of the API
+ * @param jsonKeyFile the service account json key file
+ * @param scopes the collection of OAuth scopes to use with the service account
+ * @throws GeneralSecurityException
+ * @throws IOException
+ */
+ public static RestApiHelper newInstanceWithGoogleCredential(
+ String baseUri, File jsonKeyFile, Collection<String> scopes)
+ throws GeneralSecurityException, IOException {
+ HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
+ GoogleCredential credential =
+ GoogleApiClientUtil.createCredentialFromJsonKeyFile(jsonKeyFile, scopes);
+ HttpRequestFactory requestFactory = transport.createRequestFactory(credential);
+ return new RestApiHelper(requestFactory, baseUri);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public HttpResponse execute(
+ String method, String[] uriParts, Map<String, Object> options, JSONObject data)
+ throws IOException {
+ HttpContent content = null;
+ if (data != null) {
+ content = new ByteArrayContent(JSON_MIME, data.toString().getBytes());
+ }
+ final GenericUrl uri = buildQueryUri(uriParts, options);
+ final HttpRequest request = getRequestFactory().buildRequest(method, uri, content);
+ request.setParser(new JsonObjectParser(JSON_FACTORY));
+ return request.execute();
+ }
+
+ /**
+ * Returns the HttpRequestFactory.
+ *
+ * <p>Exposed for testing.
+ */
+ @VisibleForTesting
+ public HttpRequestFactory getRequestFactory() {
+ return mRequestFactory;
+ }
+
+ /**
+ * Construct a URI for a API call with given URI parts and options. uriParts should be
+ * URL-encoded already, while options should be unencoded Strings.
+ */
+ GenericUrl buildQueryUri(String[] uriParts, Map<String, Object> options) {
+ final GenericUrl uri = new GenericUrl(mBaseUri);
+ for (int i = 0; i < uriParts.length; ++i) {
+ uri.appendRawPath(uriParts[i]);
+ // Don't add a trailing slash
+ if (i + 1 < uriParts.length) {
+ uri.appendRawPath("/");
+ }
+ }
+
+ if (options != null) {
+ uri.putAll(options);
+ }
+
+ return uri;
+ }
+}
diff --git a/tests/Android.bp b/tests/Android.bp
index c0dd39c..7d1334a 100644
--- a/tests/Android.bp
+++ b/tests/Android.bp
@@ -35,6 +35,7 @@
"tradefed",
"libprotobuf-java-full",
"truth-prebuilt",
+ "loganalysis",
],
manifest: "MANIFEST.mf",
diff --git a/tests/res/testdata/tradefed-prebuilt-cts-8.0_r21.jar b/tests/res/testdata/tradefed-prebuilt-cts-8.0_r21.jar
new file mode 100644
index 0000000..bdc8559
--- /dev/null
+++ b/tests/res/testdata/tradefed-prebuilt-cts-8.0_r21.jar
Binary files differ
diff --git a/tests/src/com/android/tradefed/FuncTests.java b/tests/src/com/android/tradefed/FuncTests.java
index 76e329d..81430b9 100644
--- a/tests/src/com/android/tradefed/FuncTests.java
+++ b/tests/src/com/android/tradefed/FuncTests.java
@@ -16,6 +16,8 @@
package com.android.tradefed;
import com.android.tradefed.build.FileDownloadCacheFuncTest;
+import com.android.tradefed.cluster.ClusterCommandLauncherFuncTest;
+import com.android.tradefed.cluster.ClusterEventUploaderFuncTest;
import com.android.tradefed.command.CommandSchedulerFuncTest;
import com.android.tradefed.command.remote.RemoteManagerFuncTest;
import com.android.tradefed.device.metric.DeviceMetricDataFuncTest;
@@ -34,6 +36,9 @@
@SuiteClasses({
// build
FileDownloadCacheFuncTest.class,
+ // cluster
+ ClusterCommandLauncherFuncTest.class,
+ ClusterEventUploaderFuncTest.class,
// command
CommandSchedulerFuncTest.class,
// command.remote
diff --git a/tests/src/com/android/tradefed/UnitTests.java b/tests/src/com/android/tradefed/UnitTests.java
index 378d513..dbb8638 100644
--- a/tests/src/com/android/tradefed/UnitTests.java
+++ b/tests/src/com/android/tradefed/UnitTests.java
@@ -16,6 +16,17 @@
package com.android.tradefed;
+import com.android.tradefed.cluster.ClusterCommandEventTest;
+import com.android.tradefed.cluster.ClusterCommandLauncherTest;
+import com.android.tradefed.cluster.ClusterCommandSchedulerTest;
+import com.android.tradefed.cluster.ClusterCommandTest;
+import com.android.tradefed.cluster.ClusterDeviceMonitorTest;
+import com.android.tradefed.cluster.ClusterEventUploaderTest;
+import com.android.tradefed.cluster.ClusterHostUtilTest;
+import com.android.tradefed.cluster.ClusterLogSaverTest;
+import com.android.tradefed.cluster.SubprocessConfigBuilderTest;
+import com.android.tradefed.cluster.SubprocessReportingHelperTest;
+import com.android.tradefed.cluster.TestOutputUploaderTest;
import com.android.tradefed.build.AppDeviceBuildInfoTest;
import com.android.tradefed.build.BootstrapBuildProviderTest;
import com.android.tradefed.build.BuildInfoTest;
@@ -361,6 +372,7 @@
import com.android.tradefed.util.testmapping.TestInfoTest;
import com.android.tradefed.util.testmapping.TestMappingTest;
import com.android.tradefed.util.zip.MergedZipEntryCollectionTest;
+import com.android.tradefed.util.RestApiHelperTest;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -390,6 +402,19 @@
// clearcut
ClearcutClientTest.class,
+ // cluster
+ ClusterCommandEventTest.class,
+ ClusterCommandLauncherTest.class,
+ ClusterCommandSchedulerTest.class,
+ ClusterCommandTest.class,
+ ClusterDeviceMonitorTest.class,
+ ClusterEventUploaderTest.class,
+ ClusterHostUtilTest.class,
+ ClusterLogSaverTest.class,
+ SubprocessConfigBuilderTest.class,
+ SubprocessReportingHelperTest.class,
+ TestOutputUploaderTest.class,
+
// command
CommandFileParserTest.class,
CommandFileWatcherTest.class,
@@ -776,6 +801,7 @@
QuotationAwareTokenizerTest.class,
RegexTrieTest.class,
RemoteZipTest.class,
+ RestApiHelperTest.class,
RunUtilTest.class,
SerializationUtilTest.class,
ShellOutputReceiverStreamTest.class,
diff --git a/tests/src/com/android/tradefed/cluster/ClusterCommandEventTest.java b/tests/src/com/android/tradefed/cluster/ClusterCommandEventTest.java
new file mode 100644
index 0000000..4e1e202
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/ClusterCommandEventTest.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import org.json.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/** Unit tests for {@link ClusterCommandEvent}. */
+@RunWith(JUnit4.class)
+public class ClusterCommandEventTest {
+
+ @Test
+ public void testToJSON() throws Exception {
+ Set<String> device_serials = new HashSet<>();
+ device_serials.add("s1");
+ ClusterCommandEvent event =
+ new ClusterCommandEvent.Builder()
+ .setType(ClusterCommandEvent.Type.InvocationStarted)
+ .setAttemptId("attempt_id")
+ .setCommandTaskId("task_id")
+ .setDeviceSerials(device_serials)
+ .setHostName("host")
+ .setTimestamp(100000)
+ .build();
+ JSONObject obj = event.toJSON();
+ Assert.assertEquals("InvocationStarted", obj.getString("type"));
+ Assert.assertEquals("task_id", obj.getString("task_id"));
+ Assert.assertEquals("attempt_id", obj.getString("attempt_id"));
+ Assert.assertEquals("host", obj.getString("hostname"));
+ Assert.assertEquals(100, obj.getLong("time"));
+ Assert.assertEquals("s1", obj.getString("device_serial"));
+ Assert.assertEquals(1, obj.getJSONArray("device_serials").length());
+ Assert.assertEquals("s1", obj.getJSONArray("device_serials").getString(0));
+ }
+
+ @Test
+ public void testToJSON_noDeviceSerial() throws Exception {
+ ClusterCommandEvent event =
+ new ClusterCommandEvent.Builder()
+ .setType(ClusterCommandEvent.Type.InvocationStarted)
+ .setAttemptId("attempt_id")
+ .setCommandTaskId("task_id")
+ .setHostName("host")
+ .setTimestamp(100000)
+ .build();
+ JSONObject obj = event.toJSON();
+ Assert.assertEquals("InvocationStarted", obj.getString("type"));
+ Assert.assertEquals("task_id", obj.getString("task_id"));
+ Assert.assertEquals("attempt_id", obj.getString("attempt_id"));
+ Assert.assertEquals("host", obj.getString("hostname"));
+ Assert.assertEquals(100, obj.getLong("time"));
+ Assert.assertFalse(obj.has("device_serial"));
+ Assert.assertEquals(0, obj.getJSONArray("device_serials").length());
+ }
+}
diff --git a/tests/src/com/android/tradefed/cluster/ClusterCommandLauncherFuncTest.java b/tests/src/com/android/tradefed/cluster/ClusterCommandLauncherFuncTest.java
new file mode 100644
index 0000000..1cbceba
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/ClusterCommandLauncherFuncTest.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.verify;
+
+import com.android.tradefed.config.Configuration;
+import com.android.tradefed.config.ConfigurationException;
+import com.android.tradefed.config.IConfiguration;
+import com.android.tradefed.config.OptionSetter;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.invoker.IInvocationContext;
+import com.android.tradefed.invoker.InvocationContext;
+import com.android.tradefed.metrics.proto.MetricMeasurement;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.TestDescription;
+import com.android.tradefed.util.FileUtil;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+
+/** Functional tests for {@link ClusterCommandLauncher}. */
+@RunWith(MockitoJUnitRunner.class)
+public class ClusterCommandLauncherFuncTest {
+
+ private static final String LEGACY_TRADEFED_JAR = "/testdata/tradefed-prebuilt-cts-8.0_r21.jar";
+ private static final String LEGACY_TRADEFED_COMMAND =
+ "host --null-device --class com.android.tradefed.device.DeviceDiagTest";
+
+ private File mRootDir;
+ private IConfiguration mConfiguration;
+ private IInvocationContext mInvocationContext;
+ private OptionSetter mOptionSetter;
+
+ @Spy private ClusterCommandLauncher mLauncher;
+ @Mock private ITestInvocationListener mListener;
+
+ @Before
+ public void setUp() throws Exception {
+ mRootDir = FileUtil.createTempDir(getClass().getName() + "_RootDir");
+ mConfiguration = new Configuration("name", "description");
+ mConfiguration.getCommandOptions().setInvocationTimeout(60_000L); // 1 minute
+ mInvocationContext = new InvocationContext();
+ mLauncher.setConfiguration(mConfiguration);
+ mLauncher.setInvocationContext(mInvocationContext);
+ mOptionSetter = new OptionSetter(mLauncher);
+ mOptionSetter.setOptionValue("cluster:root-dir", mRootDir.getAbsolutePath());
+ mOptionSetter.setOptionValue("cluster:env-var", "TF_WORK_DIR", mRootDir.getAbsolutePath());
+ }
+
+ @After
+ public void tearDown() {
+ FileUtil.recursiveDelete(mRootDir);
+ }
+
+ @Test
+ public void testRun_withLegacyTradefed()
+ throws IOException, ConfigurationException, DeviceNotAvailableException {
+ File tfJar = new File(mRootDir, "tradefed.jar");
+ FileUtil.writeToFile(getClass().getResourceAsStream(LEGACY_TRADEFED_JAR), tfJar);
+ mOptionSetter.setOptionValue("cluster:env-var", "TF_PATH", mRootDir.getAbsolutePath());
+ mOptionSetter.setOptionValue("cluster:use-subprocess-reporting", "true");
+ mOptionSetter.setOptionValue("cluster:command-line", LEGACY_TRADEFED_COMMAND);
+
+ mLauncher.run(mListener);
+
+ HashMap<String, MetricMeasurement.Metric> emptyMap = new HashMap<>();
+ verify(mListener).testRunStarted(anyString(), anyInt(), anyInt(), anyLong());
+ verify(mListener).testStarted(any(TestDescription.class), anyLong());
+ verify(mListener).testEnded(any(TestDescription.class), anyLong(), eq(emptyMap));
+ verify(mListener).testRunEnded(anyLong(), eq(emptyMap));
+ }
+}
diff --git a/tests/src/com/android/tradefed/cluster/ClusterCommandLauncherTest.java b/tests/src/com/android/tradefed/cluster/ClusterCommandLauncherTest.java
new file mode 100644
index 0000000..ddb14ea
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/ClusterCommandLauncherTest.java
@@ -0,0 +1,260 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+import com.android.loganalysis.util.ArrayUtil;
+import com.android.tradefed.config.Configuration;
+import com.android.tradefed.config.ConfigurationException;
+import com.android.tradefed.config.IConfiguration;
+import com.android.tradefed.config.OptionSetter;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.invoker.IInvocationContext;
+import com.android.tradefed.invoker.InvocationContext;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.util.CommandResult;
+import com.android.tradefed.util.CommandStatus;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.IRunUtil;
+import com.android.tradefed.util.SubprocessTestResultsParser;
+import com.android.tradefed.util.SystemUtil;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/** Unit tests for {@link ClusterCommandLauncherTest}. */
+@RunWith(JUnit4.class)
+public class ClusterCommandLauncherTest {
+
+ private static final String DEVICE_SERIAL = "device_serial";
+
+ private IRunUtil mMockRunUtil;
+ private SubprocessTestResultsParser mMockSubprocessTestResultsParser;
+ private ITestInvocationListener mMockListener;
+ private ITestDevice mMockTestDevice;
+ private File mTfPath;
+ private File mTfLibDir;
+ private File mRootDir;
+ private IConfiguration mConfiguration;
+ private IInvocationContext mInvocationContext;
+ private ClusterCommandLauncher mLauncher;
+ private OptionSetter mOptionSetter;
+
+ private File createTempDir(final String key) throws IOException {
+ return FileUtil.createTempDir(this.getClass().getName() + "_" + key);
+ }
+
+ private String[] asMatchers(String... strs) {
+ return Arrays.stream(strs).map(x -> Mockito.eq(x)).toArray(String[]::new);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ mMockRunUtil = Mockito.mock(IRunUtil.class);
+ mMockSubprocessTestResultsParser = Mockito.mock(SubprocessTestResultsParser.class);
+ mMockListener = Mockito.mock(ITestInvocationListener.class);
+ mMockTestDevice = Mockito.mock(ITestDevice.class);
+ Mockito.doReturn(DEVICE_SERIAL).when(mMockTestDevice).getSerialNumber();
+
+ mRootDir = createTempDir("RootDir");
+ mTfPath = new File(mRootDir, "TfPath");
+ mTfPath.mkdir();
+ mTfLibDir = new File(mRootDir, "TfLibDir");
+ mTfLibDir.mkdir();
+ mConfiguration = new Configuration("name", "description");
+ mConfiguration.getCommandOptions().setInvocationTimeout(10000L);
+ mInvocationContext = new InvocationContext();
+ mLauncher = Mockito.spy(ClusterCommandLauncher.class);
+ mLauncher.setConfiguration(mConfiguration);
+ mLauncher.setInvocationContext(mInvocationContext);
+ mOptionSetter = new OptionSetter(mLauncher);
+ mOptionSetter.setOptionValue("cluster:root-dir", mRootDir.getAbsolutePath());
+ mOptionSetter.setOptionValue("cluster:env-var", "TF_WORK_DIR", mRootDir.getAbsolutePath());
+ }
+
+ @After
+ public void tearDown() {
+ FileUtil.recursiveDelete(mRootDir);
+ }
+
+ @Test
+ public void testRun() throws DeviceNotAvailableException, ConfigurationException {
+ mInvocationContext.addAllocatedDevice("foo", mMockTestDevice);
+ final List<String> jars = new ArrayList<>();
+ jars.add(String.format("%s/*", mTfPath));
+ jars.add(String.format("%s/*", mTfLibDir));
+ final String classPath = ArrayUtil.join(":", jars);
+ final String tfPathValue =
+ String.format(
+ "${TF_WORK_DIR}/%s:${TF_WORK_DIR}/%s",
+ mTfPath.getName(), mTfLibDir.getName());
+ mOptionSetter.setOptionValue("cluster:env-var", "TF_PATH", tfPathValue);
+ mOptionSetter.setOptionValue("cluster:java-property", "FOO", "${TF_WORK_DIR}/foo");
+ mOptionSetter.setOptionValue("cluster:original-command-line", "original-command-line");
+ mOptionSetter.setOptionValue("cluster:command-line", "command-line");
+ final String expandedTfPathValue =
+ String.format("%s:%s", mTfPath.getAbsolutePath(), mTfLibDir.getAbsolutePath());
+ final CommandResult mockCommandResult = new CommandResult(CommandStatus.SUCCESS);
+ when(mMockRunUtil.runTimedCmd(
+ Mockito.anyLong(),
+ Mockito.<OutputStream>any(),
+ Mockito.<OutputStream>any(),
+ Mockito.<String[]>any()))
+ .thenReturn(mockCommandResult);
+ Mockito.when(mLauncher.getRunUtil()).thenReturn(mMockRunUtil);
+
+ mLauncher.run(mMockListener);
+
+ Mockito.verify(mMockRunUtil, Mockito.times(2)).setWorkingDir(mRootDir);
+ Mockito.verify(mMockRunUtil).unsetEnvVariable("TF_GLOBAL_CONFIG");
+ Mockito.verify(mMockRunUtil).setEnvVariable("TF_WORK_DIR", mRootDir.getAbsolutePath());
+ Mockito.verify(mMockRunUtil).setEnvVariable("TF_PATH", expandedTfPathValue);
+ Mockito.verify(mMockRunUtil)
+ .runTimedCmd(
+ Mockito.eq(10000L),
+ Mockito.<OutputStream>any(),
+ Mockito.<OutputStream>any(),
+ asMatchers(
+ new String[] {
+ SystemUtil.getRunningJavaBinaryPath().getAbsolutePath(),
+ "-cp",
+ classPath,
+ "-DFOO=" + mRootDir.getAbsolutePath() + "/foo",
+ "com.android.tradefed.command.CommandRunner",
+ "command-line",
+ "--serial",
+ DEVICE_SERIAL
+ }));
+ assertTrue(new File(mRootDir, "original-command-line.xml").exists());
+ }
+
+ @Test
+ public void testRun_withSetupScripts()
+ throws DeviceNotAvailableException, ConfigurationException {
+ mInvocationContext.addAllocatedDevice("foo", mMockTestDevice);
+ final String classpath = String.format("%s/*", mTfPath);
+ mOptionSetter.setOptionValue("cluster:env-var", "TF_PATH", mTfPath.getAbsolutePath());
+ mOptionSetter.setOptionValue("cluster:env-var", "FOO", "foo");
+ mOptionSetter.setOptionValue("cluster:env-var", "BAR", "bar");
+ mOptionSetter.setOptionValue("cluster:env-var", "ZZZ", "zzz");
+ mOptionSetter.setOptionValue("cluster:setup-script", "foo bar zzz");
+ mOptionSetter.setOptionValue("cluster:setup-script", "${FOO} ${BAR} ${ZZZ}");
+ mOptionSetter.setOptionValue("cluster:command-line", "command-line");
+ final CommandResult mockCommandResult = new CommandResult(CommandStatus.SUCCESS);
+ when(mMockRunUtil.runTimedCmd(
+ Mockito.anyLong(),
+ Mockito.<OutputStream>any(),
+ Mockito.<OutputStream>any(),
+ Mockito.<String[]>any()))
+ .thenReturn(mockCommandResult);
+ Mockito.when(mLauncher.getRunUtil()).thenReturn(mMockRunUtil);
+
+ mLauncher.run(mMockListener);
+
+ Mockito.verify(mMockRunUtil, Mockito.times(2)).setWorkingDir(mRootDir);
+ Mockito.verify(mMockRunUtil).unsetEnvVariable("TF_GLOBAL_CONFIG");
+ Mockito.verify(mMockRunUtil).setEnvVariable("TF_WORK_DIR", mRootDir.getAbsolutePath());
+ Mockito.verify(mMockRunUtil).setEnvVariable("TF_PATH", mTfPath.getAbsolutePath());
+ Mockito.verify(mMockRunUtil).setEnvVariable("BAR", "bar");
+ Mockito.verify(mMockRunUtil).setEnvVariable("FOO", "foo");
+ Mockito.verify(mMockRunUtil).setEnvVariable("ZZZ", "zzz");
+ Mockito.verify(mMockRunUtil, Mockito.times(2))
+ .runTimedCmd(
+ Mockito.anyLong(),
+ Mockito.<OutputStream>any(),
+ Mockito.<OutputStream>any(),
+ asMatchers(new String[] {"foo", "bar", "zzz"}));
+ Mockito.verify(mMockRunUtil)
+ .runTimedCmd(
+ Mockito.eq(10000L),
+ Mockito.<OutputStream>any(),
+ Mockito.<OutputStream>any(),
+ asMatchers(
+ new String[] {
+ SystemUtil.getRunningJavaBinaryPath().getAbsolutePath(),
+ "-cp",
+ classpath,
+ "com.android.tradefed.command.CommandRunner",
+ "command-line",
+ "--serial",
+ DEVICE_SERIAL
+ }));
+ }
+
+ @Test
+ public void testRun_withUseSubprocessReporting()
+ throws DeviceNotAvailableException, ConfigurationException, IOException {
+ mInvocationContext.addAllocatedDevice("foo", mMockTestDevice);
+ final String classpath = String.format("%s/*", mTfPath);
+ mOptionSetter.setOptionValue("cluster:env-var", "TF_PATH", mTfPath.getAbsolutePath());
+ mOptionSetter.setOptionValue("cluster:command-line", "command-line");
+ mOptionSetter.setOptionValue("cluster:use-subprocess-reporting", "true");
+ when(mMockSubprocessTestResultsParser.getSocketServerPort()).thenReturn(123);
+ Mockito.when(
+ mLauncher.createSubprocessTestResultsParser(
+ Mockito.any(), Mockito.anyBoolean(), Mockito.any()))
+ .thenReturn(mMockSubprocessTestResultsParser);
+ final CommandResult mockCommandResult = new CommandResult(CommandStatus.SUCCESS);
+ when(mMockRunUtil.runTimedCmd(
+ Mockito.anyLong(),
+ Mockito.<OutputStream>any(),
+ Mockito.<OutputStream>any(),
+ Mockito.<String[]>any()))
+ .thenReturn(mockCommandResult);
+ final File subprocessReporterConfig = new File(mRootDir, "command-line.xml");
+ Mockito.when(mLauncher.getRunUtil()).thenReturn(mMockRunUtil);
+
+ mLauncher.run(mMockListener);
+
+ String subprocessJar =
+ FileUtil.findFile(mRootDir, "subprocess-results-reporter.jar").getAbsolutePath();
+ assertTrue(subprocessReporterConfig.exists());
+ Mockito.verify(mMockRunUtil, Mockito.times(2)).setWorkingDir(mRootDir);
+ Mockito.verify(mMockRunUtil).unsetEnvVariable("TF_GLOBAL_CONFIG");
+ Mockito.verify(mMockRunUtil).setEnvVariable("TF_WORK_DIR", mRootDir.getAbsolutePath());
+ Mockito.verify(mMockRunUtil).setEnvVariable("TF_PATH", mTfPath.getAbsolutePath());
+ Mockito.verify(mMockRunUtil).unsetEnvVariable("TF_GLOBAL_CONFIG");
+ Mockito.verify(mMockRunUtil)
+ .runTimedCmd(
+ Mockito.eq(10000L),
+ Mockito.<OutputStream>any(),
+ Mockito.<OutputStream>any(),
+ asMatchers(
+ new String[] {
+ SystemUtil.getRunningJavaBinaryPath().getAbsolutePath(),
+ "-cp",
+ subprocessJar + ":" + classpath,
+ "com.android.tradefed.command.CommandRunner",
+ subprocessReporterConfig.getName(),
+ "--serial",
+ DEVICE_SERIAL,
+ }));
+ }
+}
diff --git a/tests/src/com/android/tradefed/cluster/ClusterCommandSchedulerTest.java b/tests/src/com/android/tradefed/cluster/ClusterCommandSchedulerTest.java
new file mode 100644
index 0000000..5d9c969
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/ClusterCommandSchedulerTest.java
@@ -0,0 +1,1514 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+
+import com.android.tradefed.build.IBuildInfo;
+import com.android.tradefed.command.CommandScheduler;
+import com.android.tradefed.command.remote.DeviceDescriptor;
+import com.android.tradefed.config.Configuration;
+import com.android.tradefed.config.ConfigurationException;
+import com.android.tradefed.config.ConfigurationFactory;
+import com.android.tradefed.config.GlobalConfiguration;
+import com.android.tradefed.config.IConfiguration;
+import com.android.tradefed.config.IDeviceConfiguration;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceAllocationState;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.FreeDeviceState;
+import com.android.tradefed.device.IDeviceManager;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.device.MockDeviceManager;
+import com.android.tradefed.device.NoDeviceException;
+import com.android.tradefed.device.StubDevice;
+import com.android.tradefed.invoker.IInvocationContext;
+import com.android.tradefed.invoker.InvocationContext;
+import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
+import com.android.tradefed.result.ConsoleResultReporter;
+import com.android.tradefed.result.FileInputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.result.TestDescription;
+import com.android.tradefed.targetprep.BuildError;
+import com.android.tradefed.targetprep.ITargetPreparer;
+import com.android.tradefed.targetprep.TargetSetupError;
+import com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestLogData;
+import com.android.tradefed.util.ArrayUtil;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.MultiMap;
+import com.android.tradefed.util.RunUtil;
+import com.android.tradefed.util.ZipUtil;
+import com.android.tradefed.util.keystore.IKeyStoreClient;
+import com.android.tradefed.util.keystore.StubKeyStoreClient;
+
+import com.android.tradefed.cluster.ClusterCommand.RequestType;
+import com.android.tradefed.cluster.ClusterCommandScheduler.InvocationEventHandler;
+import com.android.tradefed.util.IRestApiHelper;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpRequestFactory;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.LowLevelHttpRequest;
+import com.google.api.client.http.LowLevelHttpResponse;
+import com.google.api.client.testing.http.MockHttpTransport;
+import com.google.api.client.testing.http.MockLowLevelHttpRequest;
+import com.google.api.client.testing.http.MockLowLevelHttpResponse;
+import com.google.common.collect.ImmutableMap;
+
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.easymock.IArgumentMatcher;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.UUID;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+/** Unit tests for {@link ClusterCommandScheduler}. */
+@RunWith(JUnit4.class)
+public class ClusterCommandSchedulerTest {
+
+ private static final String CLUSTER_ID = "free_pool";
+ private static final String REQUEST_ID = "request_id";
+ private static final String COMMAND_ID = "command_id";
+ private static final String ATTEMPT_ID = "attempt_id";
+ private static final String TASK_ID = "task_id";
+ private static final String CMD_LINE = "test";
+ private static final String DEVICE_SERIAL = "serial";
+
+ @Rule public TestLogData mTestLog = new TestLogData();
+
+ private IDeviceManager mMockDeviceManager;
+ private IRestApiHelper mMockApiHelper;
+ private IClusterClient mMockClusterClient;
+ private ClusterOptions mMockClusterOptions;
+ private IClusterEventUploader<ClusterCommandEvent> mMockEventUploader;
+ private ClusterCommandScheduler mScheduler;
+ private IClusterEventUploader mMockHostUploader;
+ // Test variable to store the args of last execCommand called by CommandScheduler.
+ Stack<ArrayList<String>> mExecCmdArgs = new Stack<>();
+
+ String[] getExecCommandArgs() {
+ ArrayList<String> execCmdArgs = mExecCmdArgs.pop();
+ String[] args = new String[execCmdArgs.size()];
+ return execCmdArgs.toArray(args);
+ }
+
+ // Explicitly define this, so we can mock it
+ private static interface ICommandEventUploader
+ extends IClusterEventUploader<ClusterCommandEvent> {};
+
+ @Before
+ public void setUp() throws Exception {
+ mMockHostUploader = EasyMock.createMock(IClusterEventUploader.class);
+ mMockDeviceManager = EasyMock.createMock(IDeviceManager.class);
+ mMockApiHelper = EasyMock.createMock(IRestApiHelper.class);
+ mMockEventUploader = EasyMock.createMock(ICommandEventUploader.class);
+ mMockClusterOptions = new ClusterOptions();
+ mMockClusterOptions.setCheckFlashingPermitsLease(false);
+ mMockClusterOptions.setClusterId(CLUSTER_ID);
+ mMockClusterClient =
+ new ClusterClient() {
+ @Override
+ public IClusterEventUploader<ClusterCommandEvent> getCommandEventUploader() {
+ return mMockEventUploader;
+ }
+
+ @Override
+ public IClusterEventUploader getHostEventUploader() {
+ return mMockHostUploader;
+ }
+
+ @Override
+ public IClusterOptions getClusterOptions() {
+ return mMockClusterOptions;
+ }
+
+ @Override
+ IRestApiHelper getApiHelper() {
+ return mMockApiHelper;
+ }
+ };
+
+ mScheduler =
+ new ClusterCommandScheduler() {
+
+ @Override
+ public IClusterOptions getClusterOptions() {
+ return mMockClusterOptions;
+ }
+
+ @Override
+ IClusterClient getClusterClient() {
+ return mMockClusterClient;
+ }
+
+ @Override
+ protected IDeviceManager getDeviceManager() {
+ return mMockDeviceManager;
+ }
+
+ @Override
+ public void execCommand(IScheduledInvocationListener listener, String[] args)
+ throws ConfigurationException, NoDeviceException {
+ ArrayList<String> execCmdArgs = new ArrayList<>();
+ for (String arg : args) {
+ execCmdArgs.add(arg);
+ }
+ mExecCmdArgs.push(execCmdArgs);
+ }
+
+ @Override
+ protected boolean dryRunCommand(
+ final InvocationEventHandler handler, String[] args) {
+ return false;
+ }
+ };
+ }
+
+ private static class FakeHttpTransport extends MockHttpTransport {
+
+ private byte[] mResponseBytes;
+
+ public FakeHttpTransport(byte[] responseBytes) {
+ mResponseBytes = responseBytes;
+ }
+
+ @Override
+ public LowLevelHttpRequest buildRequest(String method, String url) throws IOException {
+ return new MockLowLevelHttpRequest() {
+ @Override
+ public LowLevelHttpResponse execute() throws IOException {
+ MockLowLevelHttpResponse response = new MockLowLevelHttpResponse();
+ response.setContent(new ByteArrayInputStream(mResponseBytes));
+ return response;
+ }
+ };
+ }
+ }
+
+ private HttpResponse buildHttpResponse(String response) throws IOException {
+ HttpRequestFactory factory =
+ new FakeHttpTransport(response.getBytes()).createRequestFactory();
+ // The method and url aren't used by our fake transport, but they can't be null
+ return factory.buildRequest("GET", new GenericUrl("http://example.com"), null).execute();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ mExecCmdArgs.clear();
+ }
+
+ private DeviceDescriptor createDevice(
+ String product, String variant, DeviceAllocationState state) {
+ return createDevice(DEVICE_SERIAL, product, variant, state);
+ }
+
+ private DeviceDescriptor createDevice(
+ String serial, String product, String variant, DeviceAllocationState state) {
+ return new DeviceDescriptor(
+ serial, false, state, product, variant, "sdkVersion", "buildId", "batteryLevel");
+ }
+
+ @Test
+ public void testGetAvailableDevices() {
+ final List<DeviceDescriptor> deviceList = new ArrayList<>();
+ deviceList.add(createDevice("product1", "variant1", DeviceAllocationState.Available));
+ deviceList.add(createDevice("product2", "variant2", DeviceAllocationState.Available));
+ deviceList.add(createDevice("product2", "variant2", DeviceAllocationState.Allocated));
+ deviceList.add(createDevice("product3", "variant3", DeviceAllocationState.Available));
+ deviceList.add(createDevice("product3", "variant3", DeviceAllocationState.Allocated));
+ deviceList.add(createDevice("product3", "variant3", DeviceAllocationState.Unavailable));
+ EasyMock.expect(mMockDeviceManager.listAllDevices()).andReturn(deviceList);
+
+ EasyMock.replay(mMockDeviceManager);
+ final MultiMap<String, DeviceDescriptor> deviceMap =
+ mScheduler.getDevices(mMockDeviceManager, false);
+ EasyMock.verify(mMockDeviceManager);
+
+ assertTrue(deviceMap.containsKey("product1:variant1"));
+ assertEquals(1, deviceMap.get("product1:variant1").size());
+ assertTrue(deviceMap.containsKey("product2:variant2"));
+ assertEquals(2, deviceMap.get("product2:variant2").size());
+ assertTrue(deviceMap.containsKey("product3:variant3"));
+ assertEquals(3, deviceMap.get("product3:variant3").size());
+ }
+
+ @Test
+ public void testGetDevices_available() {
+ final List<DeviceDescriptor> deviceList = new ArrayList<>();
+ deviceList.add(createDevice("product1", "variant1", DeviceAllocationState.Available));
+ deviceList.add(createDevice("product2", "variant2", DeviceAllocationState.Available));
+ deviceList.add(createDevice("product2", "variant2", DeviceAllocationState.Allocated));
+ deviceList.add(createDevice("product3", "variant3", DeviceAllocationState.Available));
+ deviceList.add(createDevice("product3", "variant3", DeviceAllocationState.Allocated));
+ deviceList.add(createDevice("product3", "variant3", DeviceAllocationState.Unavailable));
+ EasyMock.expect(mMockDeviceManager.listAllDevices()).andReturn(deviceList);
+
+ EasyMock.replay(mMockDeviceManager);
+ final MultiMap<String, DeviceDescriptor> deviceMap =
+ mScheduler.getDevices(mMockDeviceManager, true);
+ EasyMock.verify(mMockDeviceManager);
+
+ assertTrue(deviceMap.containsKey("product1:variant1"));
+ assertEquals(1, deviceMap.get("product1:variant1").size());
+ assertTrue(deviceMap.containsKey("product2:variant2"));
+ assertEquals(1, deviceMap.get("product2:variant2").size());
+ assertTrue(deviceMap.containsKey("product3:variant3"));
+ assertEquals(1, deviceMap.get("product3:variant3").size());
+ }
+
+ @Test
+ public void testGetDevices_LocalhostIpDevices() {
+ final List<DeviceDescriptor> deviceList = new ArrayList<>();
+ deviceList.add(
+ createDevice(
+ "127.0.0.1:101", "product1", "variant1", DeviceAllocationState.Available));
+ deviceList.add(
+ createDevice(
+ "127.0.0.1:102", "product1", "variant1", DeviceAllocationState.Available));
+ deviceList.add(createDevice("product2", "variant2", DeviceAllocationState.Allocated));
+ deviceList.add(createDevice("product3", "variant3", DeviceAllocationState.Available));
+ deviceList.add(createDevice("product3", "variant3", DeviceAllocationState.Unavailable));
+ EasyMock.expect(mMockDeviceManager.listAllDevices()).andReturn(deviceList);
+
+ EasyMock.replay(mMockDeviceManager);
+ final MultiMap<String, DeviceDescriptor> deviceMap =
+ mScheduler.getDevices(mMockDeviceManager, true);
+ EasyMock.verify(mMockDeviceManager);
+
+ assertFalse(deviceMap.containsKey("product1:variant1"));
+ assertFalse(deviceMap.containsKey("product2:variant2"));
+ assertTrue(deviceMap.containsKey("product3:variant3"));
+ assertEquals(1, deviceMap.get("product3:variant3").size());
+ }
+
+ @Test
+ public void testGetDevices_NoAvailableDevices() {
+ final List<DeviceDescriptor> deviceList = new ArrayList<>();
+ deviceList.add(createDevice("product1", "variant1", DeviceAllocationState.Allocated));
+ deviceList.add(createDevice("product2", "variant2", DeviceAllocationState.Unavailable));
+ deviceList.add(createDevice("product3", "variant3", DeviceAllocationState.Ignored));
+ EasyMock.expect(mMockDeviceManager.listAllDevices()).andReturn(deviceList);
+
+ EasyMock.replay(mMockDeviceManager);
+ final MultiMap<String, DeviceDescriptor> deviceMap =
+ mScheduler.getDevices(mMockDeviceManager, true);
+ EasyMock.verify(mMockDeviceManager);
+
+ assertTrue(deviceMap.isEmpty());
+ }
+
+ private JSONObject createCommandTask(
+ String requestId, String commandId, String taskId, String attemptId, String commandLine)
+ throws JSONException {
+
+ JSONObject ret = new JSONObject();
+ ret.put(REQUEST_ID, requestId);
+ ret.put(COMMAND_ID, commandId);
+ ret.put(ATTEMPT_ID, attemptId);
+ ret.put(TASK_ID, taskId);
+ ret.put("command_line", commandLine);
+ return ret;
+ }
+
+ private JSONObject createLeaseResponse(JSONObject... tasks) throws JSONException {
+ JSONObject response = new JSONObject();
+ JSONArray array = new JSONArray();
+ for (JSONObject task : tasks) {
+ array.put(task);
+ }
+ response.put("tasks", array);
+ return response;
+ }
+
+ @Test
+ public void testFetchHostCommands() throws Exception {
+ // Create some devices to fetch tasks for
+ mMockClusterOptions.getDeviceGroup().put("group1", "s1");
+ mMockClusterOptions.getDeviceGroup().put("group1", "s2");
+ DeviceDescriptor d1 =
+ createDevice("s1", "product1", "variant1", DeviceAllocationState.Available);
+ DeviceDescriptor d2 =
+ createDevice("s2", "product2", "variant2", DeviceAllocationState.Available);
+ DeviceDescriptor d3 =
+ createDevice("s3", "product2", "variant2", DeviceAllocationState.Available);
+ String runTarget1 = "product1:variant1";
+ String runTarget2 = "product2:variant2";
+ final MultiMap<String, DeviceDescriptor> deviceMap = new MultiMap<>();
+ deviceMap.put(runTarget1, d1);
+ deviceMap.put(runTarget2, d2);
+ deviceMap.put(runTarget2, d3);
+
+ mMockClusterOptions.getNextClusterIds().add("cluster2");
+ mMockClusterOptions.getNextClusterIds().add("cluster3");
+
+ Capture<JSONObject> capture = new Capture<>();
+ // Create some mock responses for the expected REST API calls
+ JSONObject product1Response =
+ createLeaseResponse(createCommandTask("1", "2", "3", "4", "command line 1"));
+ EasyMock.expect(
+ mMockApiHelper.execute(
+ EasyMock.eq("POST"),
+ EasyMock.aryEq(new String[] {"tasks", "leasehosttasks"}),
+ EasyMock.eq(
+ ImmutableMap.of(
+ "cluster",
+ CLUSTER_ID,
+ "hostname",
+ ClusterHostUtil.getHostName(),
+ "num_tasks",
+ Integer.toString(3))),
+ EasyMock.capture(capture)))
+ .andReturn(buildHttpResponse(product1Response.toString()));
+ // Actually fetch commands
+ EasyMock.replay(mMockApiHelper, mMockEventUploader);
+ final List<ClusterCommand> commands = mScheduler.fetchHostCommands(deviceMap);
+
+ // Verity the http request body is correct.
+ JSONArray deviceInfos = capture.getValue().getJSONArray("device_infos");
+ JSONArray clusterIds = capture.getValue().getJSONArray("next_cluster_ids");
+ assertEquals("group1", deviceInfos.getJSONObject(0).get("group_name"));
+ assertEquals("s1", deviceInfos.getJSONObject(0).get("device_serial"));
+ assertEquals("group1", deviceInfos.getJSONObject(1).get("group_name"));
+ assertEquals("s2", deviceInfos.getJSONObject(1).get("device_serial"));
+ assertFalse(deviceInfos.getJSONObject(2).has("group_name"));
+ assertEquals("s3", deviceInfos.getJSONObject(2).get("device_serial"));
+ assertEquals("cluster2", clusterIds.getString(0));
+ assertEquals("cluster3", clusterIds.getString(1));
+ // Verify that the commands were fetched
+ EasyMock.verify(mMockApiHelper, mMockEventUploader);
+ // expect 1 command allocated per device type based on availability and fetching algorithm
+ assertEquals("commands size mismatch", 1, commands.size());
+ ClusterCommand command = commands.get(0);
+ assertEquals("1", command.getRequestId());
+ assertEquals("2", command.getCommandId());
+ assertEquals("3", command.getTaskId());
+ assertEquals("4", command.getAttemptId());
+ assertEquals("command line 1", command.getCommandLine());
+ }
+
+ @Test
+ public void testFetchHostCommands_withFlashingPermitCheck() throws Exception {
+ // Create some devices to fetch tasks for
+ DeviceDescriptor d1 =
+ createDevice("s1", "product1", "variant1", DeviceAllocationState.Available);
+ String runTarget1 = "product1:variant1";
+ DeviceDescriptor d2 =
+ createDevice("s2", "product1", "variant1", DeviceAllocationState.Available);
+ final MultiMap<String, DeviceDescriptor> deviceMap = new MultiMap<>();
+ deviceMap.put(runTarget1, d1);
+ deviceMap.put(runTarget1, d2);
+
+ mMockClusterOptions.getNextClusterIds().add("cluster2");
+ mMockClusterOptions.setCheckFlashingPermitsLease(true);
+ Capture<JSONObject> capture = new Capture<>();
+ // Create some mock responses for the expected REST API calls
+ JSONObject product1Response =
+ createLeaseResponse(createCommandTask("1", "2", "3", "4", "command line 1"));
+ EasyMock.expect(
+ mMockApiHelper.execute(
+ EasyMock.eq("POST"),
+ EasyMock.aryEq(new String[] {"tasks", "leasehosttasks"}),
+ EasyMock.eq(
+ ImmutableMap.of(
+ "cluster",
+ CLUSTER_ID,
+ "hostname",
+ ClusterHostUtil.getHostName(),
+ "num_tasks",
+ Integer.toString(1))),
+ EasyMock.capture(capture)))
+ .andReturn(buildHttpResponse(product1Response.toString()));
+ EasyMock.expect(mMockDeviceManager.getAvailableFlashingPermits()).andReturn(1);
+
+ // Actually fetch commands
+ EasyMock.replay(mMockApiHelper, mMockEventUploader, mMockDeviceManager);
+ final List<ClusterCommand> commands = mScheduler.fetchHostCommands(deviceMap);
+ assertEquals(1, commands.size());
+ ClusterCommand command = commands.get(0);
+ assertEquals("1", command.getRequestId());
+ assertEquals("2", command.getCommandId());
+ assertEquals("3", command.getTaskId());
+ assertEquals("4", command.getAttemptId());
+ assertEquals("command line 1", command.getCommandLine());
+ }
+
+ /** Test when the run target pattern contains repeated patterns. */
+ @Test
+ public void testRepeatedPattern() {
+ String format = "foo-{PRODUCT}-{PRODUCT}:{PRODUCT_VARIANT}";
+ mMockClusterOptions.setRunTargetFormat(format);
+ DeviceDescriptor device =
+ new DeviceDescriptor(
+ DEVICE_SERIAL,
+ false,
+ DeviceAllocationState.Available,
+ "product",
+ "productVariant",
+ "sdkVersion",
+ "buildId",
+ "batteryLevel");
+ assertEquals(
+ "foo-product-product:productVariant",
+ ClusterHostUtil.getRunTarget(device, format, null));
+ }
+
+ /** Test default behavior when device serial is not set for command task. */
+ @Test
+ public void testExecCommandsWithNoSerials() {
+ List<ClusterCommand> cmds = new ArrayList<>();
+ ClusterCommand cmd = new ClusterCommand(COMMAND_ID, TASK_ID, CMD_LINE);
+ cmds.add(cmd);
+ mScheduler.execCommands(cmds);
+ assertEquals(CMD_LINE, cmds.get(0).getCommandLine());
+ assertArrayEquals(new String[] {CMD_LINE}, getExecCommandArgs());
+ }
+
+ /** If device serial is specified for a command task append serial to it. */
+ @Test
+ public void testExecCommandWithSerial() {
+ List<ClusterCommand> cmds = new ArrayList<>();
+ ClusterCommand cmd = new ClusterCommand(COMMAND_ID, TASK_ID, CMD_LINE);
+ cmd.setTargetDeviceSerials(ArrayUtil.list("deviceSerial"));
+ cmds.add(cmd);
+ mScheduler.execCommands(cmds);
+ assertEquals(CMD_LINE, cmds.get(0).getCommandLine());
+ assertArrayEquals(
+ new String[] {CMD_LINE, "--serial", "deviceSerial"}, getExecCommandArgs());
+ }
+
+ /** Multiple serials specified for a command task. */
+ @Test
+ public void testExecCommandWithMultipleSerials() {
+ List<ClusterCommand> cmds = new ArrayList<>();
+ ClusterCommand cmd = new ClusterCommand(COMMAND_ID, TASK_ID, CMD_LINE);
+ cmd.setTargetDeviceSerials(
+ ArrayUtil.list("deviceSerial0", "deviceSerial1", "deviceSerial2"));
+ cmds.add(cmd);
+ mScheduler.execCommands(cmds);
+ assertEquals(CMD_LINE, cmds.get(0).getCommandLine());
+ assertArrayEquals(
+ new String[] {
+ CMD_LINE,
+ "--serial",
+ "deviceSerial0",
+ "--serial",
+ "deviceSerial1",
+ "--serial",
+ "deviceSerial2"
+ },
+ getExecCommandArgs());
+ }
+
+ /** Multiple serials specified for multiple commands. */
+ @Test
+ public void testExecCommandWithMultipleCommandsAndSerials() {
+ List<String> serials = ArrayUtil.list("deviceSerial0", "deviceSerial1", "deviceSerial2");
+ List<ClusterCommand> cmds = new ArrayList<>();
+ ClusterCommand cmd0 = new ClusterCommand("command_id0", "task_id0", CMD_LINE);
+ cmd0.setTargetDeviceSerials(serials);
+ cmds.add(cmd0);
+ ClusterCommand cmd1 = new ClusterCommand("command_id1", "task_id1", "test1");
+ cmd1.setTargetDeviceSerials(serials);
+ cmds.add(cmd1);
+ mScheduler.execCommands(cmds);
+ assertEquals(CMD_LINE, cmds.get(0).getCommandLine());
+ assertEquals("test1", cmds.get(1).getCommandLine());
+ assertArrayEquals(
+ new String[] {
+ "test1",
+ "--serial",
+ "deviceSerial0",
+ "--serial",
+ "deviceSerial1",
+ "--serial",
+ "deviceSerial2"
+ },
+ getExecCommandArgs());
+ assertArrayEquals(
+ new String[] {
+ CMD_LINE,
+ "--serial",
+ "deviceSerial0",
+ "--serial",
+ "deviceSerial1",
+ "--serial",
+ "deviceSerial2"
+ },
+ getExecCommandArgs());
+ }
+
+ static ClusterCommandEvent checkClusterCommandEvent(
+ ClusterCommandEvent.Type type, Set<String> deviceSerials) {
+ EasyMock.reportMatcher(
+ new IArgumentMatcher() {
+ @Override
+ public boolean matches(Object object) {
+ if (!(object instanceof ClusterCommandEvent)) {
+ return false;
+ }
+
+ ClusterCommandEvent actual = (ClusterCommandEvent) object;
+ return (TASK_ID.equals(actual.getCommandTaskId())
+ && deviceSerials.equals(actual.getDeviceSerials())
+ && actual.getType() == type);
+ }
+
+ @Override
+ public void appendTo(StringBuffer buffer) {
+ buffer.append("checkEvent(");
+ buffer.append(type);
+ buffer.append(")");
+ }
+ });
+ return null;
+ }
+
+ static ClusterCommandEvent checkClusterCommandEvent(ClusterCommandEvent.Type type) {
+ Set<String> deviceSerials = new HashSet<String>();
+ deviceSerials.add(DEVICE_SERIAL);
+ return checkClusterCommandEvent(type, deviceSerials);
+ }
+
+ @Test
+ public void testInvocationEventHandler() {
+ ClusterCommand mockCommand = new ClusterCommand(COMMAND_ID, TASK_ID, CMD_LINE);
+ IInvocationContext context = new InvocationContext();
+ ITestDevice mockTestDevice = EasyMock.createMock(ITestDevice.class);
+ EasyMock.expect(mockTestDevice.getSerialNumber()).andReturn(DEVICE_SERIAL);
+ EasyMock.expect(mockTestDevice.getIDevice()).andReturn(new StubDevice(DEVICE_SERIAL));
+ context.addAllocatedDevice("", mockTestDevice);
+ IBuildInfo mockBuildInfo = EasyMock.createMock(IBuildInfo.class);
+ context.addDeviceBuildInfo("", mockBuildInfo);
+ ClusterCommandScheduler.InvocationEventHandler handler =
+ mScheduler.new InvocationEventHandler(mockCommand);
+
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.InvocationInitiated));
+ mMockEventUploader.flush();
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.InvocationStarted));
+ mMockEventUploader.flush();
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.TestRunInProgress));
+ EasyMock.expectLastCall().anyTimes();
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.InvocationEnded));
+ mMockEventUploader.flush();
+ Capture<ClusterCommandEvent> capture = new Capture<>();
+ mMockEventUploader.postEvent(EasyMock.capture(capture));
+ mMockEventUploader.flush();
+
+ EasyMock.replay(mMockEventUploader, mockBuildInfo, mockTestDevice);
+ handler.invocationInitiated(context);
+ handler.invocationStarted(context);
+ handler.testRunStarted("test run", 1);
+ handler.testStarted(new TestDescription("class", CMD_LINE));
+ handler.testEnded(new TestDescription("class", CMD_LINE), new HashMap<String, Metric>());
+ handler.testRunEnded(10L, new HashMap<String, Metric>());
+ handler.invocationEnded(100L);
+ context.addAllocatedDevice(DEVICE_SERIAL, mockTestDevice);
+ context.addInvocationTimingMetric(IInvocationContext.TimingEvent.FETCH_BUILD, 100L);
+ context.addInvocationTimingMetric(IInvocationContext.TimingEvent.SETUP, 200L);
+ Map<ITestDevice, FreeDeviceState> releaseMap = new HashMap<>();
+ releaseMap.put(mockTestDevice, FreeDeviceState.AVAILABLE);
+ handler.invocationComplete(context, releaseMap);
+ EasyMock.verify(mMockEventUploader, mockBuildInfo, mockTestDevice);
+ ClusterCommandEvent capturedEvent = capture.getValue();
+ assertTrue(capturedEvent.getType().equals(ClusterCommandEvent.Type.InvocationCompleted));
+ // Ensure we have not raised an unexpected error
+ assertNull(capturedEvent.getData().get(ClusterCommandEvent.DATA_KEY_ERROR));
+ assertEquals(
+ "0", capturedEvent.getData().get(ClusterCommandEvent.DATA_KEY_FAILED_TEST_COUNT));
+ assertEquals(
+ "100",
+ capturedEvent.getData().get(ClusterCommandEvent.DATA_KEY_FETCH_BUILD_TIME_MILLIS));
+ assertEquals(
+ "200", capturedEvent.getData().get(ClusterCommandEvent.DATA_KEY_SETUP_TIME_MILLIS));
+ }
+
+ /** Test that the error count is the proper one. */
+ @Test
+ public void testInvocationEventHandler_counting() {
+ ClusterCommand mockCommand = new ClusterCommand(COMMAND_ID, TASK_ID, CMD_LINE);
+ IInvocationContext context = new InvocationContext();
+ ITestDevice mockTestDevice = EasyMock.createMock(ITestDevice.class);
+ EasyMock.expect(mockTestDevice.getSerialNumber()).andReturn(DEVICE_SERIAL);
+ EasyMock.expect(mockTestDevice.getIDevice()).andReturn(new StubDevice(DEVICE_SERIAL));
+ IBuildInfo mockBuildInfo = EasyMock.createMock(IBuildInfo.class);
+ context.addAllocatedDevice("", mockTestDevice);
+ context.addDeviceBuildInfo("", mockBuildInfo);
+ ClusterCommandScheduler.InvocationEventHandler handler =
+ mScheduler.new InvocationEventHandler(mockCommand);
+
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.InvocationInitiated));
+ mMockEventUploader.flush();
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.InvocationStarted));
+ mMockEventUploader.flush();
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.TestRunInProgress));
+ EasyMock.expectLastCall().anyTimes();
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.InvocationEnded));
+ mMockEventUploader.flush();
+ Capture<ClusterCommandEvent> capture = new Capture<>();
+ mMockEventUploader.postEvent(EasyMock.capture(capture));
+ mMockEventUploader.flush();
+
+ EasyMock.replay(mMockEventUploader, mockBuildInfo, mockTestDevice);
+ handler.invocationInitiated(context);
+ handler.invocationStarted(context);
+ handler.testRunStarted("test run", 1);
+ TestDescription tid = new TestDescription("class", CMD_LINE);
+ handler.testStarted(tid);
+ handler.testFailed(tid, "failed");
+ handler.testEnded(tid, new HashMap<String, Metric>());
+ TestDescription tid2 = new TestDescription("class", "test2");
+ handler.testStarted(tid2);
+ handler.testAssumptionFailure(tid2, "I assume I failed");
+ handler.testEnded(tid2, new HashMap<String, Metric>());
+ handler.testRunEnded(10L, new HashMap<String, Metric>());
+ handler.testRunStarted("failed test run", 1);
+ TestDescription tid3 = new TestDescription("class", "test3");
+ handler.testStarted(tid3);
+ handler.testFailed(tid3, "test terminated without result");
+ handler.testRunFailed("test runner crashed");
+ handler.testRunEnded(10L, new HashMap<String, Metric>());
+ handler.invocationEnded(100L);
+ context.addAllocatedDevice(DEVICE_SERIAL, mockTestDevice);
+ Map<ITestDevice, FreeDeviceState> releaseMap = new HashMap<>();
+ releaseMap.put(mockTestDevice, FreeDeviceState.AVAILABLE);
+ handler.invocationComplete(context, releaseMap);
+ EasyMock.verify(mMockEventUploader, mockBuildInfo, mockTestDevice);
+ ClusterCommandEvent capturedEvent = capture.getValue();
+ assertTrue(capturedEvent.getType().equals(ClusterCommandEvent.Type.InvocationCompleted));
+ // Ensure we have not raised an unexpected error
+ assertNull(capturedEvent.getData().get(ClusterCommandEvent.DATA_KEY_ERROR));
+ // We only count test failure and not assumption failures.
+ assertEquals(
+ "2", capturedEvent.getData().get(ClusterCommandEvent.DATA_KEY_FAILED_TEST_COUNT));
+ assertEquals(
+ "1",
+ capturedEvent.getData().get(ClusterCommandEvent.DATA_KEY_FAILED_TEST_RUN_COUNT));
+ }
+
+ @Test
+ public void testInvocationEventHandler_longTestRun() {
+ ClusterCommand mockCommand = new ClusterCommand(COMMAND_ID, TASK_ID, CMD_LINE);
+ IInvocationContext context = new InvocationContext();
+ ITestDevice mockTestDevice = EasyMock.createMock(ITestDevice.class);
+ EasyMock.expect(mockTestDevice.getSerialNumber()).andReturn(DEVICE_SERIAL);
+ EasyMock.expect(mockTestDevice.getIDevice()).andReturn(new StubDevice(DEVICE_SERIAL));
+ context.addAllocatedDevice("", mockTestDevice);
+ IBuildInfo mockBuildInfo = EasyMock.createMock(IBuildInfo.class);
+ context.addDeviceBuildInfo("", mockBuildInfo);
+ ClusterCommandScheduler.InvocationEventHandler handler =
+ mScheduler.new InvocationEventHandler(mockCommand);
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.InvocationInitiated));
+ mMockEventUploader.flush();
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.InvocationStarted));
+ mMockEventUploader.flush();
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.TestRunInProgress));
+ EasyMock.expectLastCall().anyTimes();
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.InvocationEnded));
+ mMockEventUploader.flush();
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(ClusterCommandEvent.Type.InvocationCompleted));
+ mMockEventUploader.flush();
+
+ EasyMock.replay(mMockEventUploader, mockBuildInfo, mockTestDevice);
+ handler.invocationInitiated(context);
+ handler.invocationStarted(context);
+ handler.testRunStarted("test run", 1);
+ handler.testStarted(new TestDescription("class", CMD_LINE));
+ handler.testEnded(new TestDescription("class", CMD_LINE), new HashMap<String, Metric>());
+ handler.testRunEnded(10L, new HashMap<String, Metric>());
+ handler.invocationEnded(100L);
+ context.addAllocatedDevice(DEVICE_SERIAL, mockTestDevice);
+ Map<ITestDevice, FreeDeviceState> releaseMap = new HashMap<>();
+ releaseMap.put(mockTestDevice, FreeDeviceState.AVAILABLE);
+ handler.invocationComplete(context, releaseMap);
+ EasyMock.verify(mMockEventUploader, mockBuildInfo, mockTestDevice);
+ }
+
+ @Test
+ public void testInvocationEventHandler_multiDevice() {
+ ClusterCommand mockCommand = new ClusterCommand(COMMAND_ID, TASK_ID, CMD_LINE);
+ IInvocationContext context = new InvocationContext();
+
+ ITestDevice mockTestDevice1 = EasyMock.createMock(ITestDevice.class);
+ EasyMock.expect(mockTestDevice1.getSerialNumber()).andReturn(DEVICE_SERIAL);
+ EasyMock.expect(mockTestDevice1.getIDevice()).andReturn(new StubDevice(DEVICE_SERIAL));
+ context.addAllocatedDevice("device1", mockTestDevice1);
+ ITestDevice mockTestDevice2 = EasyMock.createMock(ITestDevice.class);
+ EasyMock.expect(mockTestDevice2.getSerialNumber()).andReturn("s2");
+ EasyMock.expect(mockTestDevice2.getIDevice()).andReturn(new StubDevice("s2"));
+ context.addAllocatedDevice("device2", mockTestDevice2);
+ IBuildInfo mockBuildInfo = EasyMock.createMock(IBuildInfo.class);
+ context.addDeviceBuildInfo("", mockBuildInfo);
+ ClusterCommandScheduler.InvocationEventHandler handler =
+ mScheduler.new InvocationEventHandler(mockCommand);
+
+ Set<String> deviceSerials = new HashSet<>();
+ deviceSerials.add(DEVICE_SERIAL);
+ deviceSerials.add("s2");
+ mMockEventUploader.postEvent(
+ checkClusterCommandEvent(
+ ClusterCommandEvent.Type.InvocationInitiated, deviceSerials));
+ mMockEventUploader.flush();
+
+ EasyMock.replay(mMockEventUploader, mockBuildInfo, mockTestDevice1, mockTestDevice2);
+ handler.invocationInitiated(context);
+ }
+
+ /**
+ * Test that when dry-run is used we validate the config and no ConfigurationException gets
+ * thrown.
+ */
+ @Test
+ public void testExecCommandsWithDryRun() {
+ mScheduler =
+ new ClusterCommandScheduler() {
+
+ @Override
+ public IClusterOptions getClusterOptions() {
+ return mMockClusterOptions;
+ }
+
+ @Override
+ IClusterClient getClusterClient() {
+ return mMockClusterClient;
+ }
+
+ @Override
+ public void execCommand(IScheduledInvocationListener listener, String[] args)
+ throws ConfigurationException, NoDeviceException {
+ ArrayList<String> execCmdArgs = new ArrayList<>();
+ for (String arg : args) {
+ execCmdArgs.add(arg);
+ }
+ mExecCmdArgs.push(execCmdArgs);
+ }
+
+ @Override
+ protected IKeyStoreClient getKeyStoreClient() {
+ return new StubKeyStoreClient();
+ }
+ };
+ ClusterCommand cmd = new ClusterCommand(COMMAND_ID, TASK_ID, "empty --dry-run");
+ mScheduler.execCommands(Arrays.asList(cmd));
+ assertEquals("empty --dry-run", cmd.getCommandLine());
+ // Nothing gets executed
+ assertTrue(mExecCmdArgs.isEmpty());
+ }
+
+ // Helper class for more functional like tests.
+ private class TestableClusterCommandScheduler extends ClusterCommandScheduler {
+
+ private IDeviceManager manager = new MockDeviceManager(1);
+ // track whether stopInvocation was called
+ private boolean stopInvocationCalled;
+
+ @Override
+ public IClusterOptions getClusterOptions() {
+ return mMockClusterOptions;
+ }
+
+ @Override
+ IClusterClient getClusterClient() {
+ return mMockClusterClient;
+ }
+
+ @Override
+ protected boolean dryRunCommand(final InvocationEventHandler handler, String[] args) {
+ return false;
+ }
+
+ @Override
+ protected IDeviceManager getDeviceManager() {
+ return manager;
+ }
+
+ // Direct getter to avoid making getDeviceManager public.
+ public IDeviceManager getTestManager() {
+ return manager;
+ }
+
+ @Override
+ protected void initLogging() {
+ // ignore
+ }
+
+ @Override
+ protected void cleanUp() {
+ // ignore
+ }
+
+ @Override
+ public boolean stopInvocation(int invocationId) {
+ return (stopInvocationCalled = true);
+ }
+
+ public boolean wasStopInvocationCalled() {
+ return stopInvocationCalled;
+ }
+ }
+
+ /** Test that when a provider returns a null build, we still handle it gracefully. */
+ @Test
+ public void testExecCommands_nullBuild() throws Exception {
+ try {
+ GlobalConfiguration.createGlobalConfiguration(new String[] {});
+ } catch (IllegalStateException e) {
+ // in case Global config is already initialized.
+ }
+ File tmpLogDir = FileUtil.createTempDir("clusterschedulertest");
+ TestableClusterCommandScheduler scheduler = new TestableClusterCommandScheduler();
+ List<ClusterCommand> cmds = new ArrayList<>();
+ // StubBuildProvider of empty.xml can return a null instead of build with --return-null
+ ClusterCommand cmd =
+ new ClusterCommand(
+ COMMAND_ID,
+ TASK_ID,
+ "empty --return-null " + "--log-file-path " + tmpLogDir.getAbsolutePath());
+ cmds.add(cmd);
+ IDeviceManager m = scheduler.getTestManager();
+ scheduler.start();
+ try {
+ // execCommands is going to allocate a device to execute the command.
+ scheduler.execCommands(cmds);
+ assertEquals(0, ((MockDeviceManager) m).getQueueOfAvailableDeviceSize());
+ assertEquals(
+ "empty --return-null --log-file-path " + tmpLogDir.getAbsolutePath(),
+ cmds.get(0).getCommandLine());
+ scheduler.shutdownOnEmpty();
+ scheduler.join(2000);
+ // Give it a bit of time as the device re-appearing can be slow.
+ RunUtil.getDefault().sleep(200L);
+ // There is only one device so allocation should succeed if device was released.
+ assertEquals(1, ((MockDeviceManager) m).getQueueOfAvailableDeviceSize());
+ ITestDevice device = m.allocateDevice();
+ assertNotNull(device);
+ } finally {
+ scheduler.shutdown();
+ File zipLog = ZipUtil.createZip(tmpLogDir);
+ try (FileInputStreamSource source = new FileInputStreamSource(zipLog, true)) {
+ mTestLog.addTestLog("testExecCommands_nullBuild", LogDataType.ZIP, source);
+ }
+ FileUtil.recursiveDelete(tmpLogDir);
+ }
+ }
+
+ /** Test that when a provider throws a build error retrieval, we still handle it gracefully. */
+ @Test
+ public void testExecCommands_buildRetrievalError() throws Exception {
+ try {
+ // we need to initialize the GlobalConfiguration when running directly in IDE.
+ GlobalConfiguration.createGlobalConfiguration(new String[] {});
+ } catch (IllegalStateException e) {
+ // in case Global config is already initialized, when running in the infra.
+ }
+ TestableClusterCommandScheduler scheduler = new TestableClusterCommandScheduler();
+ File tmpLogDir = FileUtil.createTempDir("clusterschedulertest");
+ List<ClusterCommand> cmds = new ArrayList<>();
+ // StubBuildProvider of empty.xml can throw a build error if requested via
+ // --throw-build-error
+ ClusterCommand cmd =
+ new ClusterCommand(
+ COMMAND_ID,
+ TASK_ID,
+ "empty --throw-build-error --log-file-path " + tmpLogDir.getAbsolutePath());
+ cmds.add(cmd);
+ IDeviceManager m = scheduler.getTestManager();
+ scheduler.start();
+ try {
+ scheduler.execCommands(cmds);
+ assertEquals(0, ((MockDeviceManager) m).getQueueOfAvailableDeviceSize());
+ scheduler.shutdownOnEmpty();
+ scheduler.join(5000);
+ // Give it a bit of time as the device re-appearing can be slow.
+ RunUtil.getDefault().sleep(200L);
+ // There is only one device so allocation should succeed if device was released.
+ assertEquals(1, ((MockDeviceManager) m).getQueueOfAvailableDeviceSize());
+ ITestDevice device = m.allocateDevice();
+ assertNotNull(device);
+ } finally {
+ scheduler.shutdown();
+ FileUtil.recursiveDelete(tmpLogDir);
+ }
+ }
+
+ private ClusterCommand createMockManagedCommand(int numDevices) {
+ return createMockManagedCommand(numDevices, null, null);
+ }
+
+ private ClusterCommand createMockManagedCommand(
+ int numDevices, Integer shardCount, Integer shardIndex) {
+ ClusterCommand cmd =
+ new ClusterCommand(
+ REQUEST_ID,
+ COMMAND_ID,
+ TASK_ID,
+ "command",
+ UUID.randomUUID().toString(),
+ RequestType.MANAGED,
+ shardCount,
+ shardIndex);
+ for (int i = 0; i < numDevices; i++) {
+ cmd.getTargetDeviceSerials().add(String.format("serial%d", i));
+ }
+ return cmd;
+ }
+
+ private TestEnvironment createMockTestEnvironment() {
+ TestEnvironment testEnvironment = new TestEnvironment();
+ testEnvironment.addEnvVar("ENV1", "env1");
+ testEnvironment.addEnvVar("ENV2", "env2");
+ testEnvironment.addSetupScripts("script1");
+ testEnvironment.addSetupScripts("script2");
+ testEnvironment.addJavaProperty("JAVA1", "java1");
+ testEnvironment.addJavaProperty("JAVA2", "java2");
+ testEnvironment.setOutputFileUploadUrl("output_file_upload_url");
+ testEnvironment.addOutputFilePattern("output_file_pattern1");
+ testEnvironment.addOutputFilePattern("output_file_pattern2");
+ return testEnvironment;
+ }
+
+ private List<TestResource> createMockTestResources() {
+ List<TestResource> testResources = new ArrayList<TestResource>();
+ testResources.add(new TestResource("name1", "url2"));
+ testResources.add(new TestResource("name2", "url2"));
+ return testResources;
+ }
+
+ private void verifyConfig(
+ IConfiguration config,
+ ClusterCommand cmd,
+ TestEnvironment testEnvironment,
+ List<TestResource> testResources,
+ File workDir) {
+ List<IDeviceConfiguration> deviceConfigs = config.getDeviceConfig();
+ assertEquals(cmd.getTargetDeviceSerials().size(), deviceConfigs.size());
+ for (int i = 0; i < cmd.getTargetDeviceSerials().size(); i++) {
+ String serial = cmd.getTargetDeviceSerials().get(i);
+ Collection<String> serials =
+ deviceConfigs.get(i).getDeviceRequirements().getSerials(null);
+ assertTrue(serials.size() == 1 && serials.contains(serial));
+ }
+
+ ClusterBuildProvider buildProvider =
+ (ClusterBuildProvider) deviceConfigs.get(0).getBuildProvider();
+ assertEquals(testResources.size(), buildProvider.getTestResources().size());
+ for (TestResource r : testResources) {
+ assertEquals(r.getUrl(), buildProvider.getTestResources().get(r.getName()));
+ }
+ ClusterCommandLauncher test = (ClusterCommandLauncher) config.getTests().get(0);
+ assertEquals(cmd.getCommandLine(), test.getCommandLine());
+
+ Map<String, String> envVars = new TreeMap<>();
+ envVars.put("TF_WORK_DIR", workDir.getAbsolutePath());
+ envVars.putAll(testEnvironment.getEnvVars());
+ assertEquals(envVars, test.getEnvVars());
+ assertEquals(testEnvironment.getSetupScripts(), test.getSetupScripts());
+ assertEquals(testEnvironment.getJavaProperties(), test.getJavaProperties());
+ assertEquals(testEnvironment.useSubprocessReporting(), test.useSubprocessReporting());
+ ClusterLogSaver logSaver = (ClusterLogSaver) config.getLogSaver();
+ assertEquals(cmd.getAttemptId(), logSaver.getAttemptId());
+ assertEquals(
+ String.format(
+ "%s/%s/%s/",
+ testEnvironment.getOutputFileUploadUrl(),
+ cmd.getCommandId(),
+ cmd.getAttemptId()),
+ logSaver.getOutputFileUploadUrl());
+ assertEquals(testEnvironment.getOutputFilePatterns(), logSaver.getOutputFilePatterns());
+
+ // Verify all Tradefed config objects.
+ for (TradefedConfigObject.Type type : TradefedConfigObject.Type.values()) {
+ String typeName;
+ switch (type) {
+ case TARGET_PREPARER:
+ typeName = Configuration.TARGET_PREPARER_TYPE_NAME;
+ break;
+ case RESULT_REPORTER:
+ typeName = Configuration.RESULT_REPORTER_TYPE_NAME;
+ break;
+ default:
+ continue;
+ }
+ List<Object> configObjs;
+ if (TradefedConfigObject.Type.TARGET_PREPARER.equals(type)) {
+ configObjs =
+ new ArrayList<>(
+ config.getDeviceConfig().get(0).getAllObjectOfType(typeName));
+ } else {
+ configObjs = new ArrayList<>(config.getAllConfigurationObjectsOfType(typeName));
+ }
+ for (TradefedConfigObject configDef : testEnvironment.getTradefedConfigObjects()) {
+ if (configDef.getType() != type) {
+ continue;
+ }
+ // If configObjs is empty, it means we failed find objects for some configDefs.
+ assertFalse(configObjs.isEmpty());
+ while (!configObjs.isEmpty()) {
+ Object configObj = configObjs.remove(0);
+ if (!configObj.getClass().getName().equals(configDef.getClassName())) {
+ continue;
+ }
+ }
+ }
+ }
+ }
+
+ /** Tests an execution of a managed cluster command. */
+ @Test
+ public void testExecManagedClusterCommand() throws Exception {
+ File workDir = null;
+ try {
+ ClusterCommand cmd = createMockManagedCommand(1);
+ workDir = new File(System.getProperty("java.io.tmpdir"), cmd.getAttemptId());
+ TestEnvironment testEnvironment = createMockTestEnvironment();
+ List<TestResource> testResources = createMockTestResources();
+ TestContext testContext = new TestContext();
+ mMockClusterClient = Mockito.spy(mMockClusterClient);
+ Mockito.doReturn(testEnvironment)
+ .when(mMockClusterClient)
+ .getTestEnvironment(REQUEST_ID);
+ Mockito.doReturn(testResources).when(mMockClusterClient).getTestResources(REQUEST_ID);
+ Mockito.doReturn(testContext)
+ .when(mMockClusterClient)
+ .getTestContext(REQUEST_ID, COMMAND_ID);
+ InvocationEventHandler invocationEventHandler =
+ mScheduler.new InvocationEventHandler(cmd);
+
+ mScheduler.execManagedClusterCommand(cmd, invocationEventHandler);
+
+ String[] args = getExecCommandArgs();
+ assertTrue(args.length > 0);
+ IConfiguration config =
+ ConfigurationFactory.getInstance().createConfigurationFromArgs(args);
+ verifyConfig(config, cmd, testEnvironment, testResources, workDir);
+ } finally {
+ if (workDir != null) {
+ // Clean up work directory
+ FileUtil.recursiveDelete(workDir);
+ }
+ }
+ }
+
+ /** Tests an execution of a managed cluster command for multiple devices. */
+ @Test
+ public void testExecManagedClusterCommand_multiDeviceTest() throws Exception {
+ File workDir = null;
+ try {
+ ClusterCommand cmd = createMockManagedCommand(5);
+ workDir = new File(System.getProperty("java.io.tmpdir"), cmd.getAttemptId());
+ TestEnvironment testEnvironment = createMockTestEnvironment();
+ List<TestResource> testResources = createMockTestResources();
+ TestContext testContext = new TestContext();
+ mMockClusterClient = Mockito.spy(mMockClusterClient);
+ Mockito.doReturn(testEnvironment)
+ .when(mMockClusterClient)
+ .getTestEnvironment(REQUEST_ID);
+ Mockito.doReturn(testResources).when(mMockClusterClient).getTestResources(REQUEST_ID);
+ Mockito.doReturn(testContext)
+ .when(mMockClusterClient)
+ .getTestContext(REQUEST_ID, COMMAND_ID);
+ InvocationEventHandler invocationEventHandler =
+ mScheduler.new InvocationEventHandler(cmd);
+
+ mScheduler.execManagedClusterCommand(cmd, invocationEventHandler);
+
+ String[] args = getExecCommandArgs();
+ assertTrue(args.length > 0);
+ IConfiguration config =
+ ConfigurationFactory.getInstance().createConfigurationFromArgs(args);
+ verifyConfig(config, cmd, testEnvironment, testResources, workDir);
+ } finally {
+ if (workDir != null) {
+ // Clean up work directory
+ FileUtil.recursiveDelete(workDir);
+ }
+ }
+ }
+
+ /** Tests an execution of a sharded managed cluster command. */
+ @Test
+ public void testExecManagedClusterCommand_shardedTest() throws Exception {
+ File workDir = null;
+ try {
+ ClusterCommand cmd = createMockManagedCommand(1, 100, 0);
+ workDir = new File(System.getProperty("java.io.tmpdir"), cmd.getAttemptId());
+ TestEnvironment testEnvironment = createMockTestEnvironment();
+ List<TestResource> testResources = createMockTestResources();
+ TestContext testContext = new TestContext();
+ mMockClusterClient = Mockito.spy(mMockClusterClient);
+ Mockito.doReturn(testEnvironment)
+ .when(mMockClusterClient)
+ .getTestEnvironment(REQUEST_ID);
+ Mockito.doReturn(testResources).when(mMockClusterClient).getTestResources(REQUEST_ID);
+ Mockito.doReturn(testContext)
+ .when(mMockClusterClient)
+ .getTestContext(REQUEST_ID, COMMAND_ID);
+ InvocationEventHandler invocationEventHandler =
+ mScheduler.new InvocationEventHandler(cmd);
+
+ mScheduler.execManagedClusterCommand(cmd, invocationEventHandler);
+
+ String[] args = getExecCommandArgs();
+ assertTrue(args.length > 0);
+ IConfiguration config =
+ ConfigurationFactory.getInstance().createConfigurationFromArgs(args);
+ verifyConfig(config, cmd, testEnvironment, testResources, workDir);
+ } finally {
+ if (workDir != null) {
+ // Clean up work directory
+ FileUtil.recursiveDelete(workDir);
+ }
+ }
+ }
+
+ /** Tests an execution of a managed cluster command for a IO exception case. */
+ @Test
+ public void testExecManagedClusterCommand_ioException() throws Exception {
+ ClusterCommand cmd = createMockManagedCommand(1);
+ File workDir = new File(System.getProperty("java.io.tmpdir"), cmd.getAttemptId());
+ mMockClusterClient = Mockito.spy(mMockClusterClient);
+ Mockito.doThrow(new IOException()).when(mMockClusterClient).getTestEnvironment(REQUEST_ID);
+ InvocationEventHandler invocationEventHandler = mScheduler.new InvocationEventHandler(cmd);
+
+ try {
+ mScheduler.execManagedClusterCommand(cmd, invocationEventHandler);
+ fail("IOException not thrown");
+ } catch (IOException e) {
+ }
+ assertFalse("work directory was not cleaned up", workDir.exists());
+ }
+
+ /** A mock target preparer to test injected TF config objects. */
+ public static class MockTargetPreparer implements ITargetPreparer {
+
+ @Option(name = "int", description = "An int value")
+ private int mInt;
+
+ @Option(name = "string", description = "A string value")
+ private String mString;
+
+ @Option(name = "list", description = "A list of values")
+ private List<String> mList = new ArrayList<>();
+
+ @Option(name = "map", description = "A map of key/value pairs")
+ private Map<String, String> mMap = new TreeMap<>();
+
+ @Override
+ public void setUp(ITestDevice device, IBuildInfo buildInfo)
+ throws TargetSetupError, BuildError, DeviceNotAvailableException {
+ // Do nothing
+ }
+
+ public int getInt() {
+ return mInt;
+ }
+
+ public String getString() {
+ return mString;
+ }
+
+ public List<String> getList() {
+ return mList;
+ }
+
+ public Map<String, String> getMap() {
+ return mMap;
+ }
+ }
+
+ /** Tests an execution of a managed cluster command with addition config objects. */
+ @Test
+ public void testExecManagedClusterCommand_withTradefedConfigObjects() throws Exception {
+ File workDir = null;
+ try {
+ ClusterCommand cmd = createMockManagedCommand(1);
+ workDir = new File(System.getProperty("java.io.tmpdir"), cmd.getAttemptId());
+ TestEnvironment testEnvironment = createMockTestEnvironment();
+ MultiMap<String, String> optionMap = new MultiMap<>();
+ optionMap.put("int", "1000");
+ optionMap.put("string", "foo");
+ optionMap.put("list", "foo");
+ optionMap.put("list", "bar");
+ optionMap.put("list", "zzz");
+ optionMap.put("map", "foo=bar");
+ testEnvironment.addTradefedConfigObject(
+ new TradefedConfigObject(
+ TradefedConfigObject.Type.TARGET_PREPARER,
+ MockTargetPreparer.class.getName(),
+ optionMap));
+ testEnvironment.addTradefedConfigObject(
+ new TradefedConfigObject(
+ TradefedConfigObject.Type.RESULT_REPORTER,
+ ConsoleResultReporter.class.getName(),
+ new MultiMap<>()));
+ List<TestResource> testResources = createMockTestResources();
+ TestContext testContext = new TestContext();
+ mMockClusterClient = Mockito.spy(mMockClusterClient);
+ Mockito.doReturn(testEnvironment)
+ .when(mMockClusterClient)
+ .getTestEnvironment(REQUEST_ID);
+ Mockito.doReturn(testResources).when(mMockClusterClient).getTestResources(REQUEST_ID);
+ Mockito.doReturn(testContext)
+ .when(mMockClusterClient)
+ .getTestContext(REQUEST_ID, COMMAND_ID);
+ InvocationEventHandler invocationEventHandler =
+ mScheduler.new InvocationEventHandler(cmd);
+
+ mScheduler.execManagedClusterCommand(cmd, invocationEventHandler);
+
+ String[] args = getExecCommandArgs();
+ assertTrue(args.length > 0);
+ IConfiguration config =
+ ConfigurationFactory.getInstance().createConfigurationFromArgs(args);
+ verifyConfig(config, cmd, testEnvironment, testResources, workDir);
+
+ // Verify option values.
+ Collection<Object> configObjs =
+ config.getAllConfigurationObjectsOfType(
+ Configuration.TARGET_PREPARER_TYPE_NAME);
+ MockTargetPreparer configObj =
+ (MockTargetPreparer)
+ configObjs
+ .stream()
+ .filter(
+ (obj) -> {
+ return obj instanceof MockTargetPreparer;
+ })
+ .findFirst()
+ .get();
+ assertEquals(1000, configObj.getInt());
+ assertEquals("foo", configObj.getString());
+ assertEquals(Arrays.asList("foo", "bar", "zzz"), configObj.getList());
+ assertEquals("bar", configObj.getMap().get("foo"));
+ } finally {
+ // Clean up work directory
+ FileUtil.recursiveDelete(workDir);
+ }
+ }
+
+ @Test
+ public void testShutdown_stopsHeartbeat() {
+ TestableClusterCommandScheduler scheduler = new TestableClusterCommandScheduler();
+ scheduler.start();
+ assertFalse(scheduler.getHeartbeatThreadPool().isTerminated());
+ scheduler.shutdown();
+ assertTrue(scheduler.getHeartbeatThreadPool().isTerminated());
+ }
+
+ @Test
+ public void testShutdownHard_stopsHeartbeat() {
+ TestableClusterCommandScheduler scheduler = new TestableClusterCommandScheduler();
+ scheduler.start();
+ assertFalse(scheduler.getHeartbeatThreadPool().isTerminated());
+ scheduler.shutdownHard();
+ assertTrue(scheduler.getHeartbeatThreadPool().isTerminated());
+ }
+
+ /**
+ * Ensure that we do not thrown an exception from scheduling the heartbeat after calling
+ * shutdown on the thread pool.
+ */
+ @Test
+ public void testShutdownHearbeat() throws Exception {
+ TestableClusterCommandScheduler scheduler = new TestableClusterCommandScheduler();
+ scheduler.getHeartbeatThreadPool().shutdown();
+ scheduler
+ .getHeartbeatThreadPool()
+ .scheduleAtFixedRate(
+ new Runnable() {
+ @Override
+ public void run() {
+ RunUtil.getDefault().sleep(500);
+ }
+ },
+ 0,
+ 100,
+ TimeUnit.MILLISECONDS);
+ boolean res = scheduler.getHeartbeatThreadPool().awaitTermination(5, TimeUnit.SECONDS);
+ assertTrue("HeartBeat scheduler did not terminate.", res);
+ }
+
+ /** Tests whether the checkCommandStatus option is respected. */
+ @Test
+ public void testCheckCommandState_option() {
+ TestableClusterCommandScheduler scheduler = new TestableClusterCommandScheduler();
+
+ // create new heartbeat
+ ClusterCommand command = new ClusterCommand(COMMAND_ID, TASK_ID, CMD_LINE);
+ InvocationEventHandler handler = scheduler.new InvocationEventHandler(command);
+ Runnable heartbeat = handler.new HeartbeatSender();
+
+ // populate invocation context
+ IInvocationContext context = Mockito.mock(IInvocationContext.class, RETURNS_DEEP_STUBS);
+ Mockito.when(context.getInvocationId()).thenReturn("1");
+ handler.invocationStarted(context);
+
+ // command status is CANCELED
+ mMockClusterClient = Mockito.mock(IClusterClient.class, RETURNS_DEEP_STUBS);
+ Mockito.when(mMockClusterClient.getCommandState(any(), any()))
+ .thenReturn(ClusterCommand.State.CANCELED);
+
+ // not stopped if check is disabled
+ mMockClusterOptions.setCheckCommandState(false);
+ heartbeat.run();
+ assertFalse(scheduler.wasStopInvocationCalled());
+
+ // stopped if check is enabled
+ mMockClusterOptions.setCheckCommandState(true);
+ heartbeat.run();
+ assertTrue(scheduler.wasStopInvocationCalled());
+
+ Mockito.verify(mMockClusterClient, Mockito.times(1)).getCommandState(any(), any());
+ }
+
+ /** Tests whether the heartbeat can determine the invocationId to stop. */
+ @Test
+ public void testCheckCommandState_invocationId() {
+ TestableClusterCommandScheduler scheduler = new TestableClusterCommandScheduler();
+ mMockClusterOptions.setCheckCommandState(true);
+
+ // create new heartbeat
+ ClusterCommand command = new ClusterCommand(COMMAND_ID, TASK_ID, CMD_LINE);
+ InvocationEventHandler handler = scheduler.new InvocationEventHandler(command);
+ Runnable heartbeat = handler.new HeartbeatSender();
+
+ // command status is CANCELED
+ mMockClusterClient = Mockito.mock(IClusterClient.class, RETURNS_DEEP_STUBS);
+ Mockito.when(mMockClusterClient.getCommandState(any(), any()))
+ .thenReturn(ClusterCommand.State.CANCELED);
+
+ // not stopped without invocation context
+ heartbeat.run();
+ assertFalse(scheduler.wasStopInvocationCalled());
+
+ // not stopped if invocation ID missing
+ IInvocationContext context = Mockito.mock(IInvocationContext.class, RETURNS_DEEP_STUBS);
+ handler.invocationStarted(context);
+ heartbeat.run();
+ assertFalse(scheduler.wasStopInvocationCalled());
+
+ // not stopped if invocation ID is non-numeric
+ Mockito.when(context.getInvocationId()).thenReturn("ID");
+ heartbeat.run();
+ assertFalse(scheduler.wasStopInvocationCalled());
+
+ // stopped if invocation ID is numeric
+ Mockito.when(context.getInvocationId()).thenReturn("1");
+ heartbeat.run();
+ assertTrue(scheduler.wasStopInvocationCalled());
+
+ Mockito.verify(mMockClusterClient, Mockito.times(4)).getCommandState(any(), any());
+ }
+
+ /** Tests whether the heartbeat can determine the cluster command state. */
+ @Test
+ public void testCheckCommandState_status() throws IOException {
+ TestableClusterCommandScheduler scheduler = new TestableClusterCommandScheduler();
+ mMockClusterOptions.setCheckCommandState(true);
+
+ // create new heartbeat
+ ClusterCommand command = new ClusterCommand(COMMAND_ID, TASK_ID, CMD_LINE);
+ InvocationEventHandler handler = scheduler.new InvocationEventHandler(command);
+ Runnable heartbeat = handler.new HeartbeatSender();
+
+ // populate invocation context
+ IInvocationContext context = Mockito.mock(IInvocationContext.class, RETURNS_DEEP_STUBS);
+ Mockito.when(context.getInvocationId()).thenReturn("1");
+ handler.invocationStarted(context);
+
+ mMockApiHelper = Mockito.mock(IRestApiHelper.class);
+
+ // not stopped if status is RUNNING
+ Mockito.when(mMockApiHelper.execute(any(), any(), any(), any()))
+ .thenReturn(buildHttpResponse("{\"state\": \"RUNNING\"}"));
+ heartbeat.run();
+ assertFalse(scheduler.wasStopInvocationCalled());
+
+ // not stopped if status is UNKNOWN
+ Mockito.when(mMockApiHelper.execute(any(), any(), any(), any()))
+ .thenReturn(buildHttpResponse("{\"state\": \"INVALID\"}"));
+ heartbeat.run();
+ assertFalse(scheduler.wasStopInvocationCalled());
+
+ // stopped if status is CANCELED
+ Mockito.when(mMockApiHelper.execute(any(), any(), any(), any()))
+ .thenReturn(buildHttpResponse("{\"state\": \"CANCELED\"}"));
+ heartbeat.run();
+ assertTrue(scheduler.wasStopInvocationCalled());
+
+ Mockito.verify(mMockApiHelper, Mockito.times(3)).execute(any(), any(), any(), any());
+ }
+
+ /** Tests upload events with specific host state. */
+ @Test
+ public void testUploadHostEventWithState() {
+ Capture<ClusterHostEvent> capture = new Capture<>();
+ mMockHostUploader.postEvent(EasyMock.capture(capture));
+ mMockHostUploader.flush();
+ EasyMock.replay(mMockHostUploader);
+ // Ignore exceptions here, only test uploading host states.
+ TestableClusterCommandScheduler scheduler = new TestableClusterCommandScheduler();
+ scheduler.start();
+ EasyMock.verify(mMockHostUploader);
+ ClusterHostEvent hostEvent = capture.getValue();
+ assertEquals(CLUSTER_ID, hostEvent.getClusterId());
+ assertEquals(CommandScheduler.HostState.RUNNING, hostEvent.getHostState());
+ scheduler.stop();
+ }
+}
diff --git a/tests/src/com/android/tradefed/cluster/ClusterCommandTest.java b/tests/src/com/android/tradefed/cluster/ClusterCommandTest.java
new file mode 100644
index 0000000..5c198bf
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/ClusterCommandTest.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import org.junit.Assert;
+import org.json.JSONObject;
+import org.json.JSONException;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public final class ClusterCommandTest {
+ @Test
+ public void testFromJsonWithAssignedAttemptId() throws JSONException {
+ JSONObject json = new JSONObject();
+ json.put("request_id", "i123");
+ json.put("command_id", "c123");
+ json.put("task_id", "t123");
+ json.put("command_line", "command line");
+ json.put("attempt_id", "a123");
+
+ ClusterCommand command = ClusterCommand.fromJson(json);
+
+ Assert.assertEquals("i123", command.getRequestId());
+ Assert.assertEquals("c123", command.getCommandId());
+ Assert.assertEquals("t123", command.getTaskId());
+ Assert.assertEquals("command line", command.getCommandLine());
+ Assert.assertEquals("a123", command.getAttemptId());
+ }
+
+ @Test
+ public void testFromJsonWithoutAssignedAttemptId() throws JSONException {
+ JSONObject json = new JSONObject();
+ json.put("request_id", "i123");
+ json.put("command_id", "c123");
+ json.put("task_id", "t123");
+ json.put("command_line", "command line");
+
+ ClusterCommand command = ClusterCommand.fromJson(json);
+
+ Assert.assertEquals("i123", command.getRequestId());
+ Assert.assertEquals("c123", command.getCommandId());
+ Assert.assertEquals("t123", command.getTaskId());
+ Assert.assertEquals("command line", command.getCommandLine());
+ String UUIDPattern =
+ "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$";
+ Assert.assertTrue(command.getAttemptId().matches(UUIDPattern));
+ }
+}
diff --git a/tests/src/com/android/tradefed/cluster/ClusterDeviceMonitorTest.java b/tests/src/com/android/tradefed/cluster/ClusterDeviceMonitorTest.java
new file mode 100644
index 0000000..3b11cc1
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/ClusterDeviceMonitorTest.java
@@ -0,0 +1,179 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.tradefed.command.remote.DeviceDescriptor;
+import com.android.tradefed.config.OptionSetter;
+import com.android.tradefed.util.CommandResult;
+import com.android.tradefed.util.CommandStatus;
+import com.android.tradefed.util.IRunUtil;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link ClusterDeviceMonitor}. */
+@RunWith(JUnit4.class)
+public class ClusterDeviceMonitorTest {
+
+ private static final String PRODCERTSTATUS_KEY = "LOAS status";
+ private static final String KRBSTATUS_KEY = "Kerberos status";
+ private static final String PRODCERTSTATUS_CMD = "prodcertstatus";
+ private static final String KRBSTATUS_CMD = "krbstatus";
+ private IRunUtil mRunUtil = null;
+ private ClusterDeviceMonitor mClusterDeviceMonitor = null;
+ private OptionSetter mClusterDeviceMonitorSetter = null;
+ private ClusterDeviceMonitor.EventDispatcher mEventDispatcher = null;
+ private IClusterOptions mClusterOptions = null;
+ private IClusterEventUploader<ClusterHostEvent> mHostEventUploader = null;
+ private OptionSetter mClusterOptionSetter = null;
+
+ @Before
+ public void setUp() throws Exception {
+ mRunUtil = EasyMock.createMock(IRunUtil.class);
+ mClusterOptions = new ClusterOptions();
+ mHostEventUploader = EasyMock.createMock(IClusterEventUploader.class);
+ mClusterDeviceMonitor =
+ new ClusterDeviceMonitor() {
+ @Override
+ public IRunUtil getRunUtil() {
+ return mRunUtil;
+ }
+
+ @Override
+ List<DeviceDescriptor> listDevices() {
+ return new ArrayList<DeviceDescriptor>();
+ }
+ };
+ mClusterDeviceMonitorSetter = new OptionSetter(mClusterDeviceMonitor);
+ mEventDispatcher =
+ mClusterDeviceMonitor.new EventDispatcher() {
+ @Override
+ public IClusterOptions getClusterOptions() {
+ return mClusterOptions;
+ }
+
+ @Override
+ public IClusterEventUploader<ClusterHostEvent> getEventUploader() {
+ return mHostEventUploader;
+ }
+ };
+ mClusterOptionSetter = new OptionSetter(mClusterOptions);
+ mClusterOptionSetter.setOptionValue("cluster:cluster", "cluster1");
+ mClusterOptionSetter.setOptionValue("cluster:next-cluster", "cluster2");
+ mClusterOptionSetter.setOptionValue("cluster:next-cluster", "cluster3");
+ }
+
+ @Test
+ public void testDispatch() throws Exception {
+ Capture<ClusterHostEvent> capture = new Capture<>();
+ mHostEventUploader.postEvent(EasyMock.capture(capture));
+ mHostEventUploader.flush();
+ EasyMock.replay(mHostEventUploader);
+ mEventDispatcher.dispatch();
+ EasyMock.verify(mHostEventUploader);
+ ClusterHostEvent hostEvent = capture.getValue();
+ Assert.assertEquals("cluster1", hostEvent.getClusterId());
+ Assert.assertEquals(Arrays.asList("cluster2", "cluster3"), hostEvent.getNextClusterIds());
+ }
+
+ void setOptions() throws Exception {
+ mClusterDeviceMonitorSetter.setOptionValue(
+ "host-info-cmd", PRODCERTSTATUS_KEY, PRODCERTSTATUS_CMD);
+ mClusterDeviceMonitorSetter.setOptionValue("host-info-cmd", KRBSTATUS_KEY, KRBSTATUS_CMD);
+ }
+
+ // Test getting additional host information
+ @Test
+ public void testGetAdditionalHostInfo() throws Exception {
+ setOptions();
+ String prodcertstatusOutput = "LOAS cert expires in 13h 5m";
+ CommandResult prodcertstatusMockResult = new CommandResult();
+ prodcertstatusMockResult.setStdout(prodcertstatusOutput);
+ prodcertstatusMockResult.setStatus(CommandStatus.SUCCESS);
+ EasyMock.expect(
+ mRunUtil.runTimedCmdSilently(
+ EasyMock.anyInt(), EasyMock.eq(PRODCERTSTATUS_CMD)))
+ .andReturn(prodcertstatusMockResult)
+ .times(1);
+
+ String krbstatusOutput = "android-test ticket expires in 65d 19h";
+ CommandResult krbstatusMockResult = new CommandResult();
+ krbstatusMockResult.setStdout(krbstatusOutput);
+ krbstatusMockResult.setStatus(CommandStatus.SUCCESS);
+ EasyMock.expect(mRunUtil.runTimedCmdSilently(EasyMock.anyInt(), EasyMock.eq(KRBSTATUS_CMD)))
+ .andReturn(krbstatusMockResult)
+ .times(1);
+ EasyMock.replay(mRunUtil);
+
+ Map<String, String> expected = new HashMap<>();
+ expected.put(PRODCERTSTATUS_KEY, prodcertstatusOutput);
+ expected.put(KRBSTATUS_KEY, krbstatusOutput);
+
+ Assert.assertEquals(expected, mClusterDeviceMonitor.getAdditionalHostInfo());
+ EasyMock.verify(mRunUtil);
+ }
+
+ // Test getting additional host information with no commands to run
+ @Test
+ public void testGetAdditionalHostInfo_noCommands() {
+ Map<String, String> expected = new HashMap<>();
+ Assert.assertEquals(expected, mClusterDeviceMonitor.getAdditionalHostInfo());
+ }
+
+ // Test getting additional host information with failures
+ @Test
+ public void testGetAdditionalHostInfo_commandFailed() throws Exception {
+ setOptions();
+ String prodcertstatusOutput = "LOAS cert expires in 13h 5m";
+ CommandResult prodcertstatusMockResult = new CommandResult();
+ prodcertstatusMockResult.setStdout(prodcertstatusOutput);
+ prodcertstatusMockResult.setStatus(CommandStatus.SUCCESS);
+ EasyMock.expect(
+ mRunUtil.runTimedCmdSilently(
+ EasyMock.anyInt(), EasyMock.eq(PRODCERTSTATUS_CMD)))
+ .andReturn(prodcertstatusMockResult)
+ .times(1);
+
+ String krbstatusOutput = "android-test ticket expires in 65d 19h";
+ String krbstatusError = "Some terrible failure";
+ CommandResult krbstatusMockResult = new CommandResult();
+ krbstatusMockResult.setStdout(krbstatusOutput);
+ krbstatusMockResult.setStderr(krbstatusError);
+ krbstatusMockResult.setStatus(CommandStatus.FAILED);
+ EasyMock.expect(mRunUtil.runTimedCmdSilently(EasyMock.anyInt(), EasyMock.eq(KRBSTATUS_CMD)))
+ .andReturn(krbstatusMockResult)
+ .times(1);
+ EasyMock.replay(mRunUtil);
+
+ Map<String, String> expected = new HashMap<>();
+ expected.put(PRODCERTSTATUS_KEY, prodcertstatusOutput);
+ expected.put(KRBSTATUS_KEY, krbstatusError);
+
+ Assert.assertEquals(expected, mClusterDeviceMonitor.getAdditionalHostInfo());
+ EasyMock.verify(mRunUtil);
+ }
+}
diff --git a/tests/src/com/android/tradefed/cluster/ClusterEventUploaderFuncTest.java b/tests/src/com/android/tradefed/cluster/ClusterEventUploaderFuncTest.java
new file mode 100644
index 0000000..388b7f5
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/ClusterEventUploaderFuncTest.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.android.tradefed.cluster.ClusterEventUploaderTest.Event;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/** Function tests for {@link ClusterEventUploader}. */
+@RunWith(JUnit4.class)
+public class ClusterEventUploaderFuncTest {
+
+ private ClusterEventUploader<Event> mUploader;
+
+ @Test
+ public void testPostCommandEvent_multipleThread() throws Exception {
+ final Event event1 = new Event("event1");
+ final Event event2 = new Event("event2");
+
+ final Lock lock = new ReentrantLock();
+
+ mUploader =
+ new ClusterEventUploader<Event>() {
+ @Override
+ protected void doUploadEvents(List<Event> events) throws IOException {
+ try {
+ lock.lock();
+ } finally {
+ lock.unlock();
+ }
+ }
+ };
+
+ Thread thread1 =
+ new Thread(
+ new Runnable() {
+ @Override
+ public void run() {
+ mUploader.postEvent(event1);
+ // Thread1 uses flush will be blocked.
+ mUploader.flush();
+ }
+ });
+ Thread thread2 =
+ new Thread(
+ new Runnable() {
+ @Override
+ public void run() {
+ mUploader.postEvent(event2);
+ // Thread2 doesn't use flush will not be blocked.
+ }
+ });
+ // Thread1 will be blocked. Thread2 will not be blocked.
+ lock.lock();
+ thread1.start();
+ thread2.start();
+ thread2.join(60 * 1000); // timeout in milliseconds
+ assertTrue(thread1.isAlive());
+ assertFalse(thread2.isAlive());
+ // Unblock thread 1.
+ lock.unlock();
+ thread1.join(60 * 1000); // timeout in milliseconds
+ assertFalse(thread1.isAlive());
+ }
+}
diff --git a/tests/src/com/android/tradefed/cluster/ClusterEventUploaderTest.java b/tests/src/com/android/tradefed/cluster/ClusterEventUploaderTest.java
new file mode 100644
index 0000000..e6a2c80
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/ClusterEventUploaderTest.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import static org.junit.Assert.assertEquals;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+/** Unit tests for {@link ClusterEventUploader}. */
+@RunWith(JUnit4.class)
+public class ClusterEventUploaderTest {
+
+ private IClusterEventUploader<Event> mUploader;
+ private List<Event> mUploadedEvents = null;
+
+ /** Event class used in tests. */
+ static class Event implements IClusterEvent {
+
+ String mName = null;
+ boolean mUploadSuccess = true;
+
+ public Event(String name) {
+ this(name, true);
+ }
+
+ public Event(String name, boolean uploadSuccess) {
+ mName = name;
+ mUploadSuccess = uploadSuccess;
+ }
+
+ @Override
+ public JSONObject toJSON() throws JSONException {
+ return null;
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ mUploadedEvents = new LinkedList<>();
+ mUploader =
+ new ClusterEventUploader<Event>() {
+ @Override
+ protected void doUploadEvents(List<Event> events) throws IOException {
+ for (Event e : events) {
+ if (e.mUploadSuccess) {
+ mUploadedEvents.add(e);
+ } else {
+ throw new IOException();
+ }
+ }
+ }
+ };
+ }
+
+ @Test
+ public void testPostCommandEvent_simpleEvent() {
+ // Create an event to post
+ final Event event = new Event("event");
+ mUploader.postEvent(event);
+ mUploader.flush();
+ assertEquals(1, mUploadedEvents.size());
+ assertEquals("event", mUploadedEvents.get(0).mName);
+ }
+
+ /*
+ * If there are no events in the queue, do not post.
+ */
+ @Test
+ public void testPostCommandEvent_NoEvent() {
+ mUploader.flush();
+ assertEquals(0, mUploadedEvents.size());
+ }
+
+ @Test
+ public void testPostCommandEvent_multipleEvents() {
+ final Event event = new Event("event");
+ mUploader.postEvent(event);
+ mUploader.postEvent(event);
+ mUploader.postEvent(event);
+ mUploader.postEvent(event);
+ mUploader.flush();
+ assertEquals(4, mUploadedEvents.size());
+ }
+
+ @Test
+ public void testPostCommandEvent_multipleBatches() {
+ final Event event = new Event("event");
+ mUploader.setMaxBatchSize(2);
+ mUploader.postEvent(event);
+ mUploader.postEvent(event);
+ mUploader.postEvent(event);
+ mUploader.postEvent(event);
+ mUploader.flush();
+ assertEquals(4, mUploadedEvents.size());
+ }
+
+ @Test
+ public void testPostCommandEvent_failed() {
+ final Event event = new Event("event");
+ final Event failedEvent = new Event("failedEvent", false);
+ mUploader.postEvent(event);
+ mUploader.postEvent(failedEvent);
+ mUploader.flush();
+ assertEquals(1, mUploadedEvents.size());
+ assertEquals("event", mUploadedEvents.get(0).mName);
+ failedEvent.mUploadSuccess = true;
+ mUploader.flush();
+ assertEquals(2, mUploadedEvents.size());
+ assertEquals("failedEvent", mUploadedEvents.get(1).mName);
+ }
+}
diff --git a/tests/src/com/android/tradefed/cluster/ClusterHostUtilTest.java b/tests/src/com/android/tradefed/cluster/ClusterHostUtilTest.java
new file mode 100644
index 0000000..e2c10bd
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/ClusterHostUtilTest.java
@@ -0,0 +1,290 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import com.android.ddmlib.IDevice;
+import com.android.ddmlib.IDevice.DeviceState;
+import com.android.tradefed.command.remote.DeviceDescriptor;
+import com.android.tradefed.device.DeviceAllocationState;
+import com.android.tradefed.device.DeviceManager;
+import com.android.tradefed.device.DeviceManager.FastbootDevice;
+import java.security.InvalidParameterException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link ClusterHostUtil}. */
+@RunWith(JUnit4.class)
+public class ClusterHostUtilTest {
+
+ private static final String DEVICE_SERIAL = "serial";
+
+ @Test
+ public void testIsIpPort() {
+ Assert.assertTrue(ClusterHostUtil.isIpPort("127.0.0.1:101"));
+ Assert.assertTrue(ClusterHostUtil.isIpPort("127.0.0.1"));
+ Assert.assertFalse(ClusterHostUtil.isIpPort(DEVICE_SERIAL));
+ Assert.assertFalse(ClusterHostUtil.isIpPort("127.0.0.1:notaport"));
+ }
+
+ // Test a valid TF version
+ @Test
+ public void testToValidTfVersion() {
+ String version = "12345";
+ String actual = ClusterHostUtil.toValidTfVersion(version);
+ Assert.assertEquals(version, actual);
+ }
+
+ // Test an empty TF version
+ @Test
+ public void testToValidTfVersionWithEmptyVersion() {
+ String version = "";
+ String actual = ClusterHostUtil.toValidTfVersion(version);
+ Assert.assertEquals(ClusterHostUtil.DEFAULT_TF_VERSION, actual);
+ }
+
+ // Test a null TF version
+ @Test
+ public void testToValidTfVersionWithNullVersion() {
+ String version = null;
+ String actual = ClusterHostUtil.toValidTfVersion(version);
+ Assert.assertEquals(ClusterHostUtil.DEFAULT_TF_VERSION, actual);
+ }
+
+ // Test an invalid TF version
+ @Test
+ public void testToValidTfVersionWithInvalidVersion() {
+ String version = "1abcd2efg";
+ String actual = ClusterHostUtil.toValidTfVersion(version);
+ Assert.assertEquals(ClusterHostUtil.DEFAULT_TF_VERSION, actual);
+ }
+
+ // Test default run target if nothing is specified.
+ @Test
+ public void testGetDefaultRunTarget() {
+ DeviceDescriptor device =
+ new DeviceDescriptor(
+ DEVICE_SERIAL,
+ false,
+ DeviceAllocationState.Available,
+ "product",
+ "productVariant",
+ "sdkVersion",
+ "buildId",
+ "batteryLevel");
+ Assert.assertEquals(
+ "product:productVariant", ClusterHostUtil.getRunTarget(device, null, null));
+ }
+
+ // Test default run target if nothing is specified, and product == product variant.
+ @Test
+ public void testGetDefaultRunTargetWithSameProductAndProductVariant() {
+ DeviceDescriptor device =
+ new DeviceDescriptor(
+ DEVICE_SERIAL,
+ false,
+ DeviceAllocationState.Available,
+ "product",
+ "product",
+ "sdkVersion",
+ "buildId",
+ "batteryLevel");
+ Assert.assertEquals("product", ClusterHostUtil.getRunTarget(device, null, null));
+ }
+
+ // If a constant string run target pattern is set, always return said pattern.
+ @Test
+ public void testSimpleConstantRunTargetMatchPattern() {
+ String format = "foo";
+ DeviceDescriptor device =
+ new DeviceDescriptor(
+ DEVICE_SERIAL,
+ false,
+ DeviceAllocationState.Available,
+ "product",
+ "productVariant",
+ "sdkVersion",
+ "buildId",
+ "batteryLevel");
+ Assert.assertEquals("foo", ClusterHostUtil.getRunTarget(device, format, null));
+ }
+
+ // Test run target pattern with a device tag map
+ @Test
+ public void testDeviceTagRunTargetMatchPattern_simple() {
+ String format = "{TAG}";
+ DeviceDescriptor device =
+ new DeviceDescriptor(
+ DEVICE_SERIAL,
+ false,
+ DeviceAllocationState.Available,
+ "product",
+ "productVariant",
+ "sdkVersion",
+ "buildId",
+ "batteryLevel");
+ Map<String, String> deviceTag = new HashMap<>();
+ deviceTag.put(DEVICE_SERIAL, "foo");
+ Assert.assertEquals("foo", ClusterHostUtil.getRunTarget(device, format, deviceTag));
+ }
+
+ // Test run target pattern with a device tag map, but the device serial is not in map
+ @Test
+ public void testDeviceTagRunTargetMatchPattern_missingSerial() {
+ String format = "foo{TAG}bar";
+ DeviceDescriptor device =
+ new DeviceDescriptor(
+ DEVICE_SERIAL,
+ false,
+ DeviceAllocationState.Available,
+ "product",
+ "productVariant",
+ "sdkVersion",
+ "buildId",
+ "batteryLevel");
+ Map<String, String> deviceTag = Collections.emptyMap();
+ Assert.assertEquals("foobar", ClusterHostUtil.getRunTarget(device, format, deviceTag));
+ }
+
+ // Ensure that invalid run target pattern throws an exception.
+ @Test
+ public void testInvalidRunTargetMetachPattern() {
+ String format = "foo-{INVALID PATTERN}";
+ DeviceDescriptor device =
+ new DeviceDescriptor(
+ DEVICE_SERIAL,
+ false,
+ DeviceAllocationState.Available,
+ "product",
+ "productVariant",
+ "sdkVersion",
+ "buildId",
+ "batteryLevel");
+ try {
+ ClusterHostUtil.getRunTarget(device, format, null);
+ Assert.fail("Should have thrown an InvalidParameter exception.");
+ } catch (InvalidParameterException e) {
+ // expected.
+ }
+ }
+
+ // Test all supported run target match patterns.
+ @Test
+ public void testSupportedRunTargetMatchPattern() {
+ String format = "foo-{PRODUCT}-{PRODUCT_VARIANT}-{API_LEVEL}-{DEVICE_PROP:bar}";
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+ EasyMock.expect(mockIDevice.getProperty("bar")).andReturn("zzz");
+ DeviceDescriptor device =
+ new DeviceDescriptor(
+ DEVICE_SERIAL,
+ false,
+ DeviceState.ONLINE,
+ DeviceAllocationState.Available,
+ "product",
+ "productVariant",
+ "sdkVersion",
+ "buildId",
+ "batteryLevel",
+ "",
+ "",
+ "",
+ "",
+ mockIDevice);
+ EasyMock.replay(mockIDevice);
+ Assert.assertEquals(
+ "foo-product-productVariant-sdkVersion-zzz",
+ ClusterHostUtil.getRunTarget(device, format, null));
+ EasyMock.verify(mockIDevice);
+ }
+
+ // Test all supported run target match patterns with unknown property.
+ @Test
+ public void testSupportedRunTargetMatchPattern_unknownProperty() {
+ String format = "foo-{PRODUCT}-{PRODUCT_VARIANT}-{API_LEVEL}";
+ DeviceDescriptor device =
+ new DeviceDescriptor(
+ DEVICE_SERIAL,
+ false,
+ DeviceAllocationState.Available,
+ DeviceManager.UNKNOWN_DISPLAY_STRING,
+ "productVariant",
+ "sdkVersion",
+ "buildId",
+ "batteryLevel");
+ Assert.assertEquals(
+ DeviceManager.UNKNOWN_DISPLAY_STRING,
+ ClusterHostUtil.getRunTarget(device, format, null));
+ }
+
+ /**
+ * Test PRODUCT_OR_DEVICE_CLASS that can return both product type and the stub device class.
+ * This allows an host to report both physical and stub devices.
+ */
+ @Test
+ public void testSupportedRunTargetMatchPattern_productAndStub() {
+ String format = "{PRODUCT_OR_DEVICE_CLASS}";
+ // with non-stub device we use the requested product
+ DeviceDescriptor device =
+ new DeviceDescriptor(
+ DEVICE_SERIAL,
+ false,
+ DeviceAllocationState.Available,
+ "product",
+ "productVariant",
+ "sdkVersion",
+ "buildId",
+ "batteryLevel");
+ Assert.assertEquals("product", ClusterHostUtil.getRunTarget(device, format, null));
+ // with a stub device we use the device class
+ device =
+ new DeviceDescriptor(
+ DEVICE_SERIAL,
+ true,
+ DeviceAllocationState.Available,
+ "product",
+ "productVariant",
+ "sdkVersion",
+ "buildId",
+ "batteryLevel",
+ "deviceClass",
+ "macAddress",
+ "simState",
+ "simOperator");
+ Assert.assertEquals("deviceClass", ClusterHostUtil.getRunTarget(device, format, null));
+ // with a fastboot device we use the product
+ device =
+ new DeviceDescriptor(
+ DEVICE_SERIAL,
+ true,
+ DeviceAllocationState.Available,
+ "product",
+ "productVariant",
+ "sdkVersion",
+ "buildId",
+ "batteryLevel",
+ FastbootDevice.class.getSimpleName(),
+ "macAddress",
+ "simState",
+ "simOperator");
+ Assert.assertEquals("product", ClusterHostUtil.getRunTarget(device, format, null));
+ }
+}
diff --git a/tests/src/com/android/tradefed/cluster/ClusterLogSaverTest.java b/tests/src/com/android/tradefed/cluster/ClusterLogSaverTest.java
new file mode 100644
index 0000000..9589d84
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/ClusterLogSaverTest.java
@@ -0,0 +1,342 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import com.android.tradefed.config.ConfigurationException;
+import com.android.tradefed.config.OptionSetter;
+import com.android.tradefed.invoker.InvocationContext;
+import com.android.tradefed.util.FileUtil;
+
+import com.android.tradefed.cluster.ClusterLogSaver.FilePickingStrategy;
+
+import org.json.JSONException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+/** Unit tests for {@link ClusterLogSaverTest}. */
+@RunWith(JUnit4.class)
+public class ClusterLogSaverTest {
+
+ private static final String REQUEST_ID = "request_id";
+ private static final String COMMAND_ID = "command_id";
+
+ private File mWorkDir;
+ private TestOutputUploader mMockTestOutputUploader;
+ private IClusterClient mMockClusterClient;
+ private ClusterLogSaver mClusterLogSaver;
+ private OptionSetter mOptionSetter;
+
+ @Before
+ public void setUp() throws Exception {
+ mWorkDir = FileUtil.createTempDir(this.getClass().getName());
+ mMockTestOutputUploader = Mockito.mock(TestOutputUploader.class);
+ mMockClusterClient = Mockito.mock(IClusterClient.class);
+ mClusterLogSaver = Mockito.spy(new ClusterLogSaver());
+ Mockito.doReturn(mMockTestOutputUploader).when(mClusterLogSaver).getTestOutputUploader();
+ Mockito.doReturn(mMockClusterClient).when(mClusterLogSaver).getClusterClient();
+ mOptionSetter = new OptionSetter(mClusterLogSaver);
+ mOptionSetter.setOptionValue("cluster:root-dir", mWorkDir.getAbsolutePath());
+ mOptionSetter.setOptionValue("cluster:request-id", REQUEST_ID);
+ mOptionSetter.setOptionValue("cluster:command-id", COMMAND_ID);
+ }
+
+ @After
+ public void tearDown() {
+ FileUtil.recursiveDelete(mWorkDir);
+ }
+
+ /** Create an empty file in the test's temporary directory. */
+ private File createMockFile(String path, String name) throws IOException {
+ final File dir = new File(mWorkDir, path);
+ dir.mkdirs();
+ final File file = new File(dir, name);
+ file.createNewFile();
+ return file;
+ }
+
+ /** Create an empty zip file in the test's temporary directory. */
+ private File createMockZipFile(String path, String name) throws IOException {
+ File file = createMockFile(path, name);
+ new ZipOutputStream(new FileOutputStream(file)).close();
+ return file;
+ }
+
+ /** Get the names of all entries in a zip file. */
+ private Set<String> getZipEntries(File file) throws IOException {
+ try (ZipInputStream zip = new ZipInputStream(new FileInputStream(file))) {
+ Set<String> names = new LinkedHashSet<>();
+ for (ZipEntry entry = zip.getNextEntry(); entry != null; entry = zip.getNextEntry()) {
+ names.add(entry.getName());
+ }
+ return names;
+ }
+ }
+
+ @Test
+ public void testFindTestContextFile() throws IOException {
+ final String pattern = "android-cts/results/(?<SESSION>[^/]+)/test_result\\.xml";
+ final String sessionId = "1970.01.01_00.05.10";
+ final File file =
+ createMockFile(
+ String.format("android-cts/results/%s", sessionId), "test_result.xml");
+ Map<String, String> envVars = new TreeMap<>();
+
+ File contextFile =
+ mClusterLogSaver.findTestContextFile(
+ mWorkDir, pattern, FilePickingStrategy.PICK_LAST, envVars);
+
+ assertNotNull(contextFile);
+ assertEquals(file.getAbsolutePath(), contextFile.getAbsolutePath());
+ assertEquals(1, envVars.size());
+ assertEquals(sessionId, envVars.get("SESSION"));
+ }
+
+ @Test
+ public void testFindTestContextFile_multipleMatches_pickFirst() throws IOException {
+ final String pattern = "android-cts/results/(?<SESSION>[^/]+)/test_result\\.xml";
+ final String[] sessionIds = new String[] {"1970.01.01_00.05.10", "2018.01.01_00.05.10"};
+ final File[] files =
+ new File[] {
+ createMockFile(
+ String.format("android-cts/results/%s", sessionIds[0]),
+ "test_result.xml"),
+ createMockFile(
+ String.format("android-cts/results/%s", sessionIds[1]),
+ "test_result.xml")
+ };
+ Map<String, String> envVars = new TreeMap<>();
+
+ File contextFile =
+ mClusterLogSaver.findTestContextFile(
+ mWorkDir, pattern, FilePickingStrategy.PICK_FIRST, envVars);
+
+ assertNotNull(contextFile);
+ assertEquals(files[0].getAbsolutePath(), contextFile.getAbsolutePath());
+ assertEquals(1, envVars.size());
+ assertEquals(sessionIds[0], envVars.get("SESSION"));
+ }
+
+ @Test
+ public void testFindTestContextFile_multipleMatches_pickLast() throws IOException {
+ final String pattern = "android-cts/results/(?<SESSION>[^/]+)/test_result\\.xml";
+ final String[] sessionIds = new String[] {"1970.01.01_00.05.10", "2018.01.01_00.05.10"};
+ final File[] files =
+ new File[] {
+ createMockFile(
+ String.format("android-cts/results/%s", sessionIds[0]),
+ "test_result.xml"),
+ createMockFile(
+ String.format("android-cts/results/%s", sessionIds[1]),
+ "test_result.xml")
+ };
+ Map<String, String> envVars = new TreeMap<>();
+
+ File contextFile =
+ mClusterLogSaver.findTestContextFile(
+ mWorkDir, pattern, FilePickingStrategy.PICK_LAST, envVars);
+
+ assertNotNull(contextFile);
+ assertEquals(files[1].getAbsolutePath(), contextFile.getAbsolutePath());
+ assertEquals(1, envVars.size());
+ assertEquals(sessionIds[1], envVars.get("SESSION"));
+ }
+
+ @Test
+ public void testFindTestContextFile_noMatch() throws IOException {
+ final String pattern = "android-cts/results/(?<SESSION>[^/]+)/test_result\\.xml";
+ final String sessionId = "1970.01.01_00.05.10";
+ createMockFile(String.format("android-cts/results/%s", sessionId), "test_result2.xml");
+ Map<String, String> envVars = new TreeMap<>();
+
+ File contextFile =
+ mClusterLogSaver.findTestContextFile(
+ mWorkDir, pattern, FilePickingStrategy.PICK_LAST, envVars);
+
+ assertNull(contextFile);
+ assertEquals(0, envVars.size());
+ }
+
+ @Test
+ public void testFindTestContextFile_existingEnvVar() throws IOException {
+ final String pattern = "android-cts/results/(?<SESSION>[^/]+)/test_result\\.xml";
+ final String sessionId = "1970.01.01_00.05.10";
+ final File file =
+ createMockFile(
+ String.format("android-cts/results/%s", sessionId), "test_result.xml");
+ Map<String, String> envVars = new TreeMap<>();
+ envVars.put("SESSION", "foo");
+
+ File contextFile =
+ mClusterLogSaver.findTestContextFile(
+ mWorkDir, pattern, FilePickingStrategy.PICK_LAST, envVars);
+
+ assertNotNull(contextFile);
+ assertEquals(file.getAbsolutePath(), contextFile.getAbsolutePath());
+ assertEquals(1, envVars.size());
+ assertEquals(sessionId, envVars.get("SESSION"));
+ }
+
+ @Test
+ public void testInvocationEnded() throws IOException, ConfigurationException, JSONException {
+ final InvocationContext invocationContext = new InvocationContext();
+ final String outputFileUploadUrl = "output_file_upload_url";
+ final String retryCommandLine = "retry_command_line";
+ // create output files (host_log_\d+ will be skipped)
+ File fooTxtOutputFile = createMockFile("logs", "foo.txt");
+ File fooHtmlOutputFile = createMockFile("results", "foo.html");
+ File fooCtxOutputFile = createMockFile("context", "foo.ctx");
+ createMockFile("logs", "host_log_123456.txt");
+ mOptionSetter.setOptionValue("cluster:output-file-upload-url", outputFileUploadUrl);
+ mOptionSetter.setOptionValue("cluster:output-file-pattern", ".*\\.html");
+ mOptionSetter.setOptionValue("cluster:context-file-pattern", ".*\\.ctx");
+ mOptionSetter.setOptionValue("cluster:retry-command-line", retryCommandLine);
+ final String testOutputUrl = "test_output_url";
+ Mockito.doReturn(testOutputUrl)
+ .when(mMockTestOutputUploader)
+ .uploadFile(Mockito.any(), Mockito.any());
+
+ mClusterLogSaver.invocationStarted(invocationContext);
+ mClusterLogSaver.invocationEnded(0);
+
+ // verify that file names are properly stored including any path prefix
+ final File fileNamesFile = new File(mWorkDir, ClusterLogSaver.FILE_NAMES_FILE_NAME);
+ String expectedFileNames =
+ Stream.of("tool-logs/foo.txt", "foo.html", "foo.ctx")
+ .sorted()
+ .collect(Collectors.joining("\n"));
+ assertEquals(expectedFileNames, FileUtil.readStringFromFile(fileNamesFile));
+
+ Mockito.verify(mMockTestOutputUploader).setUploadUrl(outputFileUploadUrl);
+ Mockito.verify(mMockTestOutputUploader).uploadFile(fileNamesFile, null);
+ Mockito.verify(mMockTestOutputUploader)
+ .uploadFile(fooTxtOutputFile, ClusterLogSaver.TOOL_LOG_PATH);
+ Mockito.verify(mMockTestOutputUploader).uploadFile(fooHtmlOutputFile, null);
+ Mockito.verify(mMockTestOutputUploader).uploadFile(fooCtxOutputFile, null);
+ TestContext expTextContext = new TestContext();
+ expTextContext.addTestResource(new TestResource("context/foo.ctx", testOutputUrl));
+ expTextContext.setCommandLine(retryCommandLine);
+ Mockito.verify(mMockClusterClient)
+ .updateTestContext(REQUEST_ID, COMMAND_ID, expTextContext);
+ }
+
+ @Test
+ public void testInvocationEnded_uploadError() throws IOException, ConfigurationException {
+ final InvocationContext invocationContext = new InvocationContext();
+ final String outputFileUploadUrl = "output_file_upload_url";
+ createMockFile("", ClusterLogSaver.FILE_NAMES_FILE_NAME);
+ File fooTxtOutputFile = createMockFile("logs", "foo.txt");
+ File barTxtOutputFile = createMockFile("logs", "bar.txt");
+ File fooCtxOutputFile = createMockFile("context", "foo.ctx");
+ mOptionSetter.setOptionValue("cluster:context-file-pattern", ".*\\.ctx");
+ mOptionSetter.setOptionValue("cluster:output-file-upload-url", outputFileUploadUrl);
+ final String testOutputUrl = "test_output_url";
+ Mockito.doReturn(testOutputUrl)
+ .doThrow(RuntimeException.class)
+ .doReturn(testOutputUrl)
+ .when(mMockTestOutputUploader)
+ .uploadFile(Mockito.any(), Mockito.any());
+
+ mClusterLogSaver.invocationStarted(invocationContext);
+ mClusterLogSaver.invocationEnded(0);
+
+ // verify that file names are properly stored including any path prefix
+ final File fileNamesFile = new File(mWorkDir, ClusterLogSaver.FILE_NAMES_FILE_NAME);
+ String expectedFileNames =
+ Stream.of("tool-logs/foo.txt", "tool-logs/bar.txt", "foo.ctx")
+ .sorted()
+ .collect(Collectors.joining("\n"));
+ assertEquals(expectedFileNames, FileUtil.readStringFromFile(fileNamesFile));
+
+ Mockito.verify(mMockTestOutputUploader).setUploadUrl(outputFileUploadUrl);
+ Mockito.verify(mMockTestOutputUploader).uploadFile(fileNamesFile, null);
+ Mockito.verify(mMockTestOutputUploader)
+ .uploadFile(fooTxtOutputFile, ClusterLogSaver.TOOL_LOG_PATH);
+ Mockito.verify(mMockTestOutputUploader)
+ .uploadFile(barTxtOutputFile, ClusterLogSaver.TOOL_LOG_PATH);
+ Mockito.verify(mMockTestOutputUploader).uploadFile(fooCtxOutputFile, null);
+ }
+
+ @Test
+ public void testInvocationEnded_duplicateUpload() throws IOException, ConfigurationException {
+ String outputFileUploadUrl = "output_file_upload_url";
+ mOptionSetter.setOptionValue("cluster:output-file-upload-url", outputFileUploadUrl);
+
+ // create two files with same destination, only the second should get picked
+ mOptionSetter.setOptionValue("cluster:output-file-pattern", ".*\\.txt");
+ mOptionSetter.setOptionValue("cluster:file-picking-strategy", "PICK_LAST");
+ File first = createMockFile("first", "foo.txt");
+ File second = createMockFile("second", "foo.txt");
+
+ InvocationContext invocationContext = new InvocationContext();
+ mClusterLogSaver.invocationStarted(invocationContext);
+ mClusterLogSaver.invocationEnded(0);
+
+ // verify that only one file is recorded in the filename file
+ File fileNamesFile = new File(mWorkDir, ClusterLogSaver.FILE_NAMES_FILE_NAME);
+ assertEquals("foo.txt", FileUtil.readStringFromFile(fileNamesFile));
+
+ // verify that only the second file is uploaded (and filename file)
+ Mockito.verify(mMockTestOutputUploader).setUploadUrl(outputFileUploadUrl);
+ Mockito.verify(mMockTestOutputUploader).uploadFile(fileNamesFile, null);
+ Mockito.verify(mMockTestOutputUploader).uploadFile(second, null);
+ Mockito.verifyNoMoreInteractions(mMockTestOutputUploader);
+ }
+
+ @Test
+ public void testAppendFilesToContext() throws IOException {
+ // create context file
+ File contextFile = createMockZipFile("context", "context.zip");
+
+ // create files to append
+ File first = createMockFile("extra", "1.txt");
+ File second = createMockFile("extra", "2.txt");
+
+ // append files using absolute, relative, and unknown paths
+ mClusterLogSaver.appendFilesToContext(
+ contextFile, Arrays.asList(first.getAbsolutePath(), "extra/2.txt", "unknown.txt"));
+
+ // verify that context file contains the additional files
+ Set<String> entries = getZipEntries(contextFile);
+ assertTrue(entries.contains("1.txt"));
+ assertTrue(entries.contains("2.txt"));
+ assertFalse(entries.contains("unknown.txt")); // unknown file ignored
+ }
+}
diff --git a/tests/src/com/android/tradefed/cluster/SubprocessConfigBuilderTest.java b/tests/src/com/android/tradefed/cluster/SubprocessConfigBuilderTest.java
new file mode 100644
index 0000000..e588940
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/SubprocessConfigBuilderTest.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import com.android.tradefed.result.LegacySubprocessResultsReporter;
+import com.android.tradefed.util.FileUtil;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import java.io.File;
+import java.io.IOException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+/** Unit tests for {@link SubprocessConfigBuilder} */
+@RunWith(JUnit4.class)
+public class SubprocessConfigBuilderTest {
+
+ private static final String REPORTER_CLASS = LegacySubprocessResultsReporter.class.getName();
+
+ private SubprocessConfigBuilder mConfigBuilder;
+ private File mWorkDir;
+
+ @Before
+ public void setUp() throws IOException {
+ mConfigBuilder = new SubprocessConfigBuilder();
+ mWorkDir = FileUtil.createTempDir("tfjar");
+ }
+
+ @After
+ public void tearDown() {
+ FileUtil.recursiveDelete(mWorkDir);
+ }
+
+ @Test
+ public void testCreateWrapperConfig() throws Exception {
+ String oriConfigName = "testConfig";
+ String reporterPort = "1024";
+ mConfigBuilder
+ .setWorkingDir(mWorkDir)
+ .setOriginalConfig(oriConfigName)
+ .setPort(reporterPort);
+ File config = mConfigBuilder.build();
+ assertNotNull(config);
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.parse(config);
+ verifyWrapperXml(doc, oriConfigName, reporterPort);
+ }
+
+ private void verifyWrapperXml(Document doc, String oriConfigName, String reporterPort) {
+ NodeList inc = doc.getElementsByTagName("include");
+ assertEquals(1, inc.getLength());
+ String incName = ((Element) inc.item(0)).getAttribute("name");
+ assertEquals(oriConfigName, incName);
+ NodeList reporter = doc.getElementsByTagName("result_reporter");
+ assertEquals(1, reporter.getLength());
+ String reporterClass = ((Element) reporter.item(0)).getAttribute("class");
+ assertEquals(REPORTER_CLASS, reporterClass);
+ NodeList option = ((Element) reporter.item(0)).getElementsByTagName("option");
+ assertEquals(1, option.getLength());
+ String optionName = ((Element) option.item(0)).getAttribute("name");
+ assertEquals("subprocess-report-port", optionName);
+ String optionPort = ((Element) option.item(0)).getAttribute("value");
+ assertEquals(reporterPort, optionPort);
+ }
+}
diff --git a/tests/src/com/android/tradefed/cluster/SubprocessReportingHelperTest.java b/tests/src/com/android/tradefed/cluster/SubprocessReportingHelperTest.java
new file mode 100644
index 0000000..f507b83
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/SubprocessReportingHelperTest.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import static org.junit.Assert.*;
+
+import com.android.tradefed.result.LegacySubprocessResultsReporter;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.ZipUtil2;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.File;
+import java.io.IOException;
+
+/** Unit tests for {@link SubprocessReportingHelper} */
+@RunWith(JUnit4.class)
+public class SubprocessReportingHelperTest {
+ private SubprocessReportingHelper mHelper;
+ private SubprocessConfigBuilder mConfigBuilder;
+ private String reporterClassName;
+ private File mWorkDir;
+
+ @Before
+ public void setUp() throws IOException {
+ mHelper = new SubprocessReportingHelper();
+ mConfigBuilder = new SubprocessConfigBuilder();
+ reporterClassName = LegacySubprocessResultsReporter.class.getName();
+ mWorkDir = FileUtil.createTempDir("tfjar");
+ }
+
+ @After
+ public void tearDown() {
+ FileUtil.recursiveDelete(mWorkDir);
+ }
+
+ @Test
+ public void testCreateSubprocessReporterJar() throws IOException {
+ File jar = mHelper.createSubprocessReporterJar(mWorkDir);
+ assertNotNull(jar);
+ File extractedJar = ZipUtil2.extractZipToTemp(jar, "tmp-jar");
+ try {
+ assertNotNull(FileUtil.findFile(extractedJar, "LegacySubprocessResultsReporter.class"));
+ assertNotNull(FileUtil.findFile(extractedJar, "SubprocessTestResultsParser.class"));
+ assertNotNull(FileUtil.findFile(extractedJar, "SubprocessEventHelper.class"));
+ assertNotNull(FileUtil.findFile(extractedJar, "SubprocessResultsReporter.class"));
+ } finally {
+ FileUtil.recursiveDelete(extractedJar);
+ }
+ }
+
+ @Test
+ public void testGetNewCommandLine() throws IOException {
+ String oldCommandLine = "cts arg1 arg2 arg3";
+ String newCommandLine = mHelper.buildNewCommandConfig(oldCommandLine, "1024", mWorkDir);
+ assertNotNull(FileUtil.findFile(mWorkDir, "cts.xml"));
+ assertEquals("cts.xml arg1 arg2 arg3", newCommandLine);
+ }
+
+ @Test
+ public void testGetNewCommandLine_withSlashes() throws IOException {
+ String oldCommandLine = "foo/bar/cts arg1 arg2 arg3";
+ String newCommandLine = mHelper.buildNewCommandConfig(oldCommandLine, "1024", mWorkDir);
+ assertNotNull(FileUtil.findFile(mWorkDir, "foo\\$bar\\$cts.xml"));
+ assertEquals("foo$bar$cts.xml arg1 arg2 arg3", newCommandLine);
+ }
+}
diff --git a/tests/src/com/android/tradefed/cluster/TestOutputUploaderTest.java b/tests/src/com/android/tradefed/cluster/TestOutputUploaderTest.java
new file mode 100644
index 0000000..13cab2b
--- /dev/null
+++ b/tests/src/com/android/tradefed/cluster/TestOutputUploaderTest.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.cluster;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.IRunUtil;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+
+/** Unit tests for {@link TestOutputUploader}. */
+@RunWith(JUnit4.class)
+public class TestOutputUploaderTest {
+
+ private IRunUtil mMockRunUtil;
+ private TestOutputUploader mTestOutputUploader;
+ private File mOutputFile;
+
+ @Before
+ public void setUp() throws IOException {
+ mMockRunUtil = Mockito.mock(IRunUtil.class);
+ mTestOutputUploader =
+ new TestOutputUploader() {
+ @Override
+ IRunUtil getRunUtil() {
+ return mMockRunUtil;
+ }
+ };
+ mOutputFile = File.createTempFile(this.getClass().getName(), ".log");
+ }
+
+ @After
+ public void tearDown() {
+ mOutputFile.delete();
+ }
+
+ @Test
+ public void testUploadFile_fileProtocol_rootPath() throws IOException {
+ File destFolder = null;
+ try {
+ destFolder = FileUtil.createTempDir(this.getClass().getName());
+ final String uploadUrl = "file://" + destFolder.getAbsolutePath();
+ mTestOutputUploader.setUploadUrl(uploadUrl);
+
+ final String outputFileUrl = mTestOutputUploader.uploadFile(mOutputFile, null);
+
+ assertEquals(uploadUrl + "/" + mOutputFile.getName(), outputFileUrl);
+ final File uploadedFile = new File(new URL(outputFileUrl).getPath());
+ assertTrue(uploadedFile.exists());
+ assertTrue(FileUtil.compareFileContents(mOutputFile, uploadedFile));
+ } finally {
+ if (destFolder != null) {
+ FileUtil.recursiveDelete(destFolder);
+ }
+ }
+ }
+
+ @Test
+ public void testUploadFile_fileProtocol_withDestPath() throws IOException {
+ File destRootFolder = null;
+ try {
+ destRootFolder = FileUtil.createTempDir(this.getClass().getName());
+ final String uploadUrl = "file://" + destRootFolder.getAbsolutePath();
+ mTestOutputUploader.setUploadUrl(uploadUrl);
+
+ final String outputFileUrl = mTestOutputUploader.uploadFile(mOutputFile, "sub_dir");
+
+ assertEquals(uploadUrl + "/" + "sub_dir" + "/" + mOutputFile.getName(), outputFileUrl);
+ final File uploadedFile = new File(new URL(outputFileUrl).getPath());
+ assertTrue(uploadedFile.exists());
+ assertTrue(FileUtil.compareFileContents(mOutputFile, uploadedFile));
+ } finally {
+ if (destRootFolder != null) {
+ FileUtil.recursiveDelete(destRootFolder);
+ }
+ }
+ }
+}
diff --git a/tests/src/com/android/tradefed/util/RestApiHelperTest.java b/tests/src/com/android/tradefed/util/RestApiHelperTest.java
new file mode 100644
index 0000000..c96d006
--- /dev/null
+++ b/tests/src/com/android/tradefed/util/RestApiHelperTest.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.util;
+
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpRequestFactory;
+import com.google.api.client.http.javanet.NetHttpTransport;
+
+import junit.framework.TestCase;
+
+import java.util.Collections;
+
+/** Unit tests for {@link RestApiHelper}. */
+public class RestApiHelperTest extends TestCase {
+ private static final String BASE_URI = "https://www.googleapis.com/test/";
+
+ private RestApiHelper mHelper = null;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ HttpRequestFactory requestFactory = new NetHttpTransport().createRequestFactory();
+ mHelper = new RestApiHelper(requestFactory, BASE_URI);
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ public void testBuildQueryUri() {
+ String[] uriParts = {"add", "new", "fox"};
+ GenericUrl uri = new GenericUrl(BASE_URI + "add/new/fox");
+
+ assertEquals(uri, mHelper.buildQueryUri(uriParts, Collections.<String, Object>emptyMap()));
+ }
+}
